diff --git a/src/Plugin/migrate_plus/data_parser/Json.php b/src/Plugin/migrate_plus/data_parser/Json.php index 586da02..769c8e1 100755 --- a/src/Plugin/migrate_plus/data_parser/Json.php +++ b/src/Plugin/migrate_plus/data_parser/Json.php @@ -44,13 +44,15 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa * * @param string $url * URL of a JSON feed. + * @param string $item_selector + * Selector within the data content at which useful data is found. * * @return array * The selected data to be iterated. * * @throws \GuzzleHttp\Exception\RequestException */ - protected function getSourceData($url) { + protected function getSourceData($url, $item_selector) { // Use cached source data if this is the first request or URL is same as the // last time we made the request. if ($this->currentUrl != $url || !$this->sourceData) { @@ -59,8 +61,8 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa // Convert objects to associative arrays. $this->sourceData = json_decode($response, TRUE); - // If json_decode() returned NULL, it may be that the data is not valid - // utf-8 (see http://php.net/manual/en/function.json-decode.php#86997). + // If json_decode() has returned NULL, it might be that the data isn't + // valid utf8 - see http://php.net/manual/en/function.json-decode.php#86997. if (is_null($this->sourceData)) { $utf8response = utf8_encode($response); $this->sourceData = json_decode($utf8response, TRUE); @@ -69,19 +71,19 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa } // Backwards-compatibility for depth selection. - if (is_int($this->itemSelector)) { + if (is_int($item_selector)) { return $this->selectByDepth($this->sourceData); } // Otherwise, we're using xpath-like selectors. - $selectors = explode('/', trim($this->itemSelector, '/')); - $source_data = $this->sourceData; + $selectors = explode('/', trim($item_selector, '/')); + $return = $this->sourceData; foreach ($selectors as $selector) { if (!empty($selector)) { - $source_data = $source_data[$selector]; + $return = $return[$selector]; } } - return $source_data; + return $return; } /** @@ -120,13 +122,13 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa */ protected function openSourceUrl($url) { // (Re)open the provided URL. - $source_data = $this->getSourceData($url); - // Only proceed if $source_data is an array or object. - if (is_array($source_data) || is_object($source_data)) { - $this->iterator = new \ArrayIterator($source_data); - return TRUE; + $source_data = $this->getSourceData($url, $this->itemSelector); + // Ensure there is source data at the current url. + if (is_null($source_data)) { + return FALSE; } - return FALSE; + $this->iterator = new \ArrayIterator($source_data); + return TRUE; } /** @@ -161,114 +163,121 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa protected function getNextUrls($url) { $next_urls = []; - if (!empty($this->configuration['pager']['type'])) { - $data = $this->getSourceData($url); - if ($this->configuration['pager']['type'] == 'urls' && !empty($data)) { - if (is_array($data)) { - $next_urls = $data; - } - else { - $next_urls[] = $data; + // If a pager selector is provided, get the data from the source. + $selector_data = []; + if (!empty($this->configuration['pager']['selector'])) { + $selector_data = $this->getSourceData($url, $this->configuration['pager']['selector']); + } + + // Logic for each type of pager. + switch ($this->configuration['pager']['type']) { + case 'urls': + if (!empty($selector_data)) { + if (is_array($selector_data)) { + $next_urls = $selector_data; + } + else { + $next_urls[] = $selector_data; + } } - } - elseif ($this->configuration['pager']['type'] == 'cursor') { - if ($data && is_scalar($data)) { + break; + + case 'cursor': + if ($selector_data && is_scalar($selector_data)) { // Just use 'cursor' as a default parameter key if not provided. $key = !empty($this->configuration['pager']['key']) ? $this->configuration['pager']['key'] : 'cursor'; - // Parse the url. + // Parse the url and replace the cursor param value and rebuild the url. $path = UrlHelper::parse($url); - // Replace the cursor param value and rebuild the url. - $path['query'][$key] = $data; + $path['query'][$key] = $selector_data; $next_urls[] = Url::fromUri($path['path'], [ 'query' => $path['query'], 'fragment' => $path['fragment'], ])->toString(); } - } - elseif ($this->configuration['pager']['type'] == 'page') { - if ($data && is_scalar($data)) { + break; + + case 'page': + if ($selector_data && is_scalar($selector_data)) { // Just use 'page' as a default parameter key if not provided. $key = !empty($this->configuration['pager']['key']) ? $this->configuration['pager']['key'] : 'page'; - // Parse the url. + // Parse the url and replace the cursor param value and rebuild the url. $path = UrlHelper::parse($url); - // Replace the page param value and rebuild the url. - $path['query'][$key] = $data + 1; + $path['query'][$key] = $selector_data + 1; $next_urls[] = Url::fromUri($path['path'], [ 'query' => $path['query'], 'fragment' => $path['fragment'], ])->toString(); } - } - if ($this->configuration['pager']['type'] == 'paginator') { - if ($data && !empty($data)) { - // The first pass uses the endpoint's default size. - // @todo Handle first URL set page size on first pass. - if (!isset($this->configuration['pager']['default_num_items'])) { - throw new MigrateException('Pager "default_num_items" must be configured.'); - } - $num_items = $this->configuration['pager']['default_num_items']; + break; + + case 'paginator': + // The first pass uses the endpoint's default size. + // @todo Handle first URL set page size on first pass. + if (!isset($this->configuration['pager']['default_num_items'])) { + throw new MigrateException('Pager "default_num_items" must be configured.'); + } + $num_items = $this->configuration['pager']['default_num_items']; - // Use 'page' as a default page parameter key if not provided. - $page_key = !empty($this->configuration['pager']['page_key']) ? $this->configuration['pager']['page_key'] : 'page'; + // Use 'page' as a default page parameter key if not provided. + $page_key = !empty($this->configuration['pager']['page_key']) ? $this->configuration['pager']['page_key'] : 'page'; - // Set default paginator type. - $paginator_type_options = ['page_number', 'starting_item']; - $paginator_type = $paginator_type_options[0]; - // Check configured paginator type. - if (!empty($this->configuration['pager']['paginator_type'])) { - if (!in_array($this->configuration['pager']['paginator_type'], $paginator_type_options)) { - // Not set to one of the two available options. - throw new MigrateException( - 'Pager "paginator_type" must be configured as either "page_number" or "starting_item" ("page_number" is default).' - ); - } - $paginator_type = $this->configuration['pager']['paginator_type']; + // Set default paginator type. + $paginator_type_options = ['page_number', 'starting_item']; + $paginator_type = $paginator_type_options[0]; + // Check configured paginator type. + if (!empty($this->configuration['pager']['paginator_type'])) { + if (!in_array($this->configuration['pager']['paginator_type'], $paginator_type_options)) { + // Not set to one of the two available options. + throw new MigrateException( + 'Pager "paginator_type" must be configured as either "page_number" or "starting_item" ("page_number" is default).' + ); } + $paginator_type = $this->configuration['pager']['paginator_type']; + } - // Use 'pagesize' as a default page parameter key if not provided. - $size_key = !empty($this->configuration['pager']['size_key']) ? $this->configuration['pager']['size_key'] : 'pagesize'; + // Use 'pagesize' as a default page parameter key if not provided. + $size_key = !empty($this->configuration['pager']['size_key']) ? $this->configuration['pager']['size_key'] : 'pagesize'; - // Parse the url. - $path = UrlHelper::parse($url); + // Parse the url. + $path = UrlHelper::parse($url); - $curr_page = !empty($path['query'][$page_key]) ? $path['query'][$page_key] : 0; + $curr_page = !empty($path['query'][$page_key]) ? $path['query'][$page_key] : 0; - // @todo Use core's QueryBase and pager. - // @see contrib module external_entities \Entity\Query\External\Query.php for example. - $next_start = $curr_page + $num_items; - $next_end = $num_items; - // Use "page_number" when the pager uses page numbers to determine - // the item to start at, use "starting_item" when the pager uses the - // item number to start at. - if ($paginator_type === 'page_number') { - $next_start = $curr_page + 1; - } + // @todo Use core's QueryBase and pager. + // @see contrib module external_entities \Entity\Query\External\Query.php for example. + $next_start = $curr_page + $num_items; + $next_end = $num_items; + // Use "page_number" when the pager uses page numbers to determine + // the item to start at, use "starting_item" when the pager uses the + // item number to start at. + if ($paginator_type === 'page_number') { + $next_start = $curr_page + 1; + } - // Replace the paginator param value. - $path['query'][$page_key] = $next_start; - // Replace the size param value. - $path['query'][$size_key] = $next_end; + // Replace the paginator param value. + $path['query'][$page_key] = $next_start; + // Replace the size param value. + $path['query'][$size_key] = $next_end; - // Rebuild the url. - $next_urls[] = Url::fromUri($path['path'], [ - 'query' => $path['query'], - 'fragment' => $path['fragment'], - ])->toString(); + // Rebuild the url. + $next_urls[] = Url::fromUri($path['path'], [ + 'query' => $path['query'], + 'fragment' => $path['fragment'], + ])->toString(); - // Service may return 404 for last page, ensure next_urls are valid. - foreach ($next_urls as $key => $next_url) { - try { - $response = $this->getDataFetcherPlugin()->getResponse($next_url); - if ($response->getStatusCode() !== 200) { - unset($next_urls[$key]); - } - } - catch (\Exception $e) { + // Service may return 404 for last page, ensure next_urls are valid. + foreach ($next_urls as $key => $next_url) { + try { + $response = $this->getDataFetcherPlugin()->getResponse($next_url); + if ($response->getStatusCode() !== 200) { unset($next_urls[$key]); } } + catch (\Exception $e) { + unset($next_urls[$key]); + } } - } + break; } return array_merge(parent::getNextUrls($url), $next_urls);