From 01fb4eba1358d8acd1fe029ae257ef7b63416ace Mon Sep 17 00:00:00 2001 From: doidd Date: Fri, 19 Nov 2021 13:10:11 +0700 Subject: [PATCH] 2887450-55 --- README.md | 32 ++ composer.json | 7 + drush.services.yml | 6 + src/Annotation/FormatManipulator.php | 23 + src/Commands/ViewsDataExportCommands.php | 396 ++++++++++++++++++ src/FormatManipulatorDefault.php | 51 +++ src/FormatManipulatorInterface.php | 29 ++ src/FormatManipulatorLoader.php | 72 ++++ .../FormatManipulatorCsv.php | 23 + .../FormatManipulatorJson.php | 33 ++ .../FormatManipulatorXlsx.php | 48 +++ .../FormatManipulatorXml.php | 35 ++ src/Plugin/QueueWorker/FileWriter.php | 14 + src/Plugin/QueueWorker/FileWriterBase.php | 80 ++++ .../Functional/ViewsDataExportBatchTest.php | 4 +- views_data_export.services.yml | 4 + 16 files changed, 855 insertions(+), 2 deletions(-) create mode 100644 drush.services.yml create mode 100644 src/Annotation/FormatManipulator.php create mode 100644 src/Commands/ViewsDataExportCommands.php create mode 100644 src/FormatManipulatorDefault.php create mode 100644 src/FormatManipulatorInterface.php create mode 100644 src/FormatManipulatorLoader.php create mode 100644 src/Plugin/FormatManipulator/FormatManipulatorCsv.php create mode 100644 src/Plugin/FormatManipulator/FormatManipulatorJson.php create mode 100644 src/Plugin/FormatManipulator/FormatManipulatorXlsx.php create mode 100644 src/Plugin/FormatManipulator/FormatManipulatorXml.php create mode 100644 src/Plugin/QueueWorker/FileWriter.php create mode 100644 src/Plugin/QueueWorker/FileWriterBase.php create mode 100644 views_data_export.services.yml diff --git a/README.md b/README.md index d583eb5..0dd3724 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,38 @@ CONFIGURATION downloaded without a file association. 9. Navigate to the path of the View Page to generate the report. +DRUSH INTEGRATION +----------------- +A command is provided to execute a views_data_export display of a view and writes the output to file, provided into the +third parameter. + +###### Command + +`views-data-export` + +###### Alias +`vde` + +###### Parameters +1. *view_name* (The name of the view) +2. *display_id* (display id of data_export display) +3. *filename* (filename to write the result of a command. Could be absolute path into the system. If no absolute path + provided file will be saved into project directory) + +**BEWARE**: If file already exists it will be overwritten. + +###### Usage examples +`drush views-data-export my_view_name views_data_export_display_id output.csv` + +`drush vde my_view_name views_data_export_display_id output.csv` + +###### Errors +The command throws `\Exception` in the next cases: +1. *my_view_name* does not exist; +2. *my_view_name* does not have provided *display_id*; +3. *display_id* does not exist; +4. Unable to create file *filename* into provided directory. In this case you have to check directory permissions or + provide valid path. MAINTAINERS ----------- diff --git a/composer.json b/composer.json index e9d1be6..b742f80 100644 --- a/composer.json +++ b/composer.json @@ -9,5 +9,12 @@ "require-dev": { "drupal/search_api": "~1.12", "drupal/xls_serialization": "~1.0" + }, + "extra": { + "drush": { + "services": { + "drush.services.yml": "^9" + } + } } } diff --git a/drush.services.yml b/drush.services.yml new file mode 100644 index 0000000..60e8a38 --- /dev/null +++ b/drush.services.yml @@ -0,0 +1,6 @@ +services: + views_data_export.commands: + class: \Drupal\views_data_export\Commands\ViewsDataExportCommands + arguments: ['@account_switcher', '@plugin.manager.format_manipulator', '@queue', '@plugin.manager.queue_worker'] + tags: + - { name: drush.command } diff --git a/src/Annotation/FormatManipulator.php b/src/Annotation/FormatManipulator.php new file mode 100644 index 0000000..bc0a09d --- /dev/null +++ b/src/Annotation/FormatManipulator.php @@ -0,0 +1,23 @@ +accountSwitcher = $account_switcher; + $this->formatManipulatorLoader = $formatManipulatorLoader; + $this->queueFactory = $queueFactory->get('views_data_export_queue'); + $this->queueManager = $queue_manager; + } + + /** + * Implements views_data_export command arguments validation. + * + * @hook validate views-data-export + */ + public function viewsDataExportValidate(CommandData $commandData) { + // Extract argument values. + $input = $commandData->input(); + $view_name = $input->getArgument('view_name'); + $display_id = $input->getArgument('display_id'); + $output_file = $input->getArgument('output_file'); + $options = $commandData->options(); + + // Verify view existence. + $view = Views::getView($view_name); + if (!is_object($view)) { + throw new \Exception(dt('The view !view does not exist.', ['!view' => $view_name])); + } + + // Verify existence of the display. + if (empty($view->setDisplay($display_id))) { + throw new \Exception(dt('The view !view does not have the !display display.', [ + '!view' => $view_name, + '!display' => $display_id, + ])); + } + + // Verify the display type. + $view_display = $view->getDisplay(); + if ($view_display->getPluginId() !== 'data_export') { + throw new \Exception(dt('Incorrect display_id provided, expected a views data export display, found !display instead.', [ + '!display' => $view_display->getPluginId(), + ])); + } + + // Handle relative paths. + $output_path = []; + preg_match('/(.*\/)*([^\/]*)$/', $output_file, $output_path); + + // Attempt to resolve the directory. + $output_path[1] = realpath($output_path[1]); + if (empty($output_path[1])) { + throw new \Exception('No such directory.'); + } + + // Validate filename. + if (empty($output_path[2])) { + // Set default filename. + $output_path[2] = implode('_', [ + 'views_export', + $view_name, + $display_id, + ]); + + $this->logger()->notice(dt('No file name has been provided, using "!default" instead.', [ + '!default' => $output_path[2], + ])); + } + + // Validate filename extension. + if (strpos($output_path[2], '.') === FALSE) { + // Extract current style format. + $export_format = reset($view_display->getOption('style')['options']['formats']); + + // Apply output file extension. + $output_path[2] = StringUtils::interpolate('!filename.!format', [ + '!filename' => $output_path[2], + '!format' => $export_format, + ]); + + $this->logger()->notice(dt('No file format has been provided, using "!format" instead.', [ + '!format' => $export_format, + ])); + } + + // Update the output file path. + $input->setArgument('output_file', implode('/', [ + $output_path[1], $output_path[2], + ])); + } + + /** + * Executes views_data_export display of a view and writes the output to file. + * + * @param string $view_name + * The name of the view. + * @param string $display_id + * The id of the views_data_export display to execute on the view. + * @param string $output_file + * The file to write the results to - will be overwritten if it already + * exists. + * + * @usage views-data-export my_view_name views_data_export_display_id + * output.csv Export my_view_name:views_data_export_display_id and write the + * output to output.csv in the current directory. + * + * @command views-data-export + * @aliases vde + * + * @throws \Exception + * If view does not exist. + */ + public function viewsDataExport($view_name, $display_id, $output_file) { + $view = Views::getView($view_name); + $view->setDisplay($display_id); + + // Switch to root user (--user option was removed from drush 9). + $this->accountSwitcher->switchTo(new UserSession(['uid' => 1])); + + if ($this->isBatched($view)) { + $this->performBatchExport($view, $output_file); + $this->logger()->success(dt( + 'Data export saved to !output_file', + ['!output_file' => $output_file] + )); + } + else { + @ob_end_clean(); + ob_start(); + // This export isn't batched. + $res = $view->executeDisplay($display_id); + // Get the results, and clean the output buffer. + // Get the results, and clean the output buffer. + echo $res["#markup"] ? ($res["#markup"] instanceof MarkupInterface ? $res["#markup"]->__toString() : $res["#markup"]) : ''; + // Ensure the data are not empty, otherwise file_put_contents() throws + // an error. + echo "\n"; + + // Save the results to file. + // Copy file over. + if (file_put_contents($output_file, ob_get_clean())) { + $this->logger() + ->success(dt('Data export saved to !output_file', [ + '!output_file' => $output_file, + ])); + } + else { + throw new \Exception(dt('The file could not be copied to the selected destination')); + } + } + + // Switch account back. + $this->accountSwitcher->switchBack(); + } + + /** + * Determine if this view should run as a batch or not. + * + * @param \Drupal\views\ViewExecutable $view + * View to check if it's batched. + * + * @return bool + * TRUE if view is batched, FALSE otherwise. + */ + public function isBatched(ViewExecutable $view) { + return ($view->display_handler->getOption('export_method') == 'batch') + && empty($view->live_preview); + } + + /** + * Performs batch exporting routine. + * + * @param \Drupal\views\ViewExecutable $view + * View the data of which must be exported. + * @param string $output_file + * Output file path. + * + * @throws \Drupal\Core\Queue\SuspendQueueException + * @throws \Exception + */ + private function performBatchExport(ViewExecutable $view, $output_file) { + $handler = $view->getDisplay(); + $view->preExecute(); + $view->build(); + + $items_per_batch = $handler->getOption('export_batch_size'); + $export_limit = $handler->getOption('export_limit'); + $export_count = 0; + + // Perform total rows count query separately from the view. + $count_query = clone $view->query; + $count_query = $count_query->query(TRUE)->execute(); + $count_query->allowRowCount = TRUE; + $export_items = $count_query->rowCount(); + + // Apply export limit. + if (!empty($export_limit)) { + $export_items = min($export_items, $export_limit); + } + + $output_format = reset($view->getStyle()->options['formats']); + $format_manipulator = $this->formatManipulatorLoader->createInstance($output_format); + + // Disable both views and entity storage cache before executing the + // rendering procedures. + $this->entityCacheDisable($view); + + $queue = $this->queueFactory; + + // We need to multiply on $items_per_batch and substract + // $items_per_batch to get the number close to $export_items. + // Because the last item in $queue->numberOfItems() could be + // lesser than $items_per_batch and by acting this way we ensure + // that we have the exact number of operations and don't have + // any duplicates. For sure there could be duplicates if we + // repopulate queue, but it won't do any harm to an export + // or at least it shouldn't. + $total_queue_items = $queue->numberOfItems() * $items_per_batch - $items_per_batch; + + // In case if queue was filled multiple times with the same + // items we must delete queue and start a new one. + if ($total_queue_items > $export_items) { + $queue->deleteQueue(); + } + + // If there are some items in queue ask if to create new queue or + // to keep this one. + if ($queue->numberOfItems() > 0 + && $this->confirm(dt( + 'There are @items items in queue. Do you want to create new queue(y) or use existing one(n)?', + ['@items' => $queue->numberOfItems()] + ))) { + $queue->deleteQueue(); + } + + $proceed = FALSE; + // If there are some items in queue ask if to populate queue or + // to execute existing. + if ($queue->numberOfItems() > 0) { + $proceed = $this->confirm(dt( + 'Do you want to execute them(y) or proceed to populating queue(n)?' + )); + } + + // If there is no items in queue or previous answer was to populate + // queue(n) we must populate queue. + if (!$proceed) { + $this->logger()->info(dt('Adding data to queue...')); + + // Perform per chunk view rendering. + while ($export_count < $export_items) { + $queue->createItem([ + 'view' => $view, + 'export_count' => $export_count, + 'items_per_batch' => $items_per_batch, + 'format_manipulator' => $format_manipulator, + 'output_file' => $output_file, + 'export_items' => $export_items, + ]); + + // Shift rendering start position. + $export_count += $items_per_batch; + } + } + + $this->logger()->info(dt('Queue filled. Starting executing items')); + $queue_worker = $this->queueManager->createInstance('views_data_export_queue'); + + while ($item = $queue->claimItem()) { + if ($item->data['export_count'] !== $export_items) { + try { + $queue_worker->processItem($item->data); + + $this->logger()->info(dt('Exporting records !from to !to.', [ + '!from' => $item->data['export_count'], + '!to' => $item->data['export_count'] + $item->data['items_per_batch'], + ])); + + $queue->deleteItem($item); + } + catch (SuspendQueueException $e) { + $queue->releaseItem($item); + break; + } + catch (\Exception $e) { + $this->logger()->error(dt('An error occured. !error', [ + '!error' => $e, + ])); + } + } + } + + $this->logger()->info(dt('Executed')); + $queue->deleteQueue(); + } + + /** + * Disables entity related views cache. + * + * @param \Drupal\views\ViewExecutable $view + * A view object the cache for which must be disabled. + */ + private function entityCacheDisable(ViewExecutable &$view) { + $entity_types = $view->query->getEntityTableInfo(); + $entity_type_manager = \Drupal::entityTypeManager(); + + foreach ($entity_types as $entity_type => $entity_description) { + try { + $entity_type_definition = $entity_type_manager->getDefinition($entity_type); + + // Set the static cache flag to false. + $entity_type_definition->set('static_cache', FALSE); + } catch (PluginNotFoundException $e) {} + } + + // Disable views cache plugin. + $handler = $view->getDisplay(); + $handler_cache_none = [ + 'type' => 'none', + 'options' => [], + ]; + + $handler->setOption('cache', $handler_cache_none); + $handler->options['cache'] = $handler_cache_none; + } + +} diff --git a/src/FormatManipulatorDefault.php b/src/FormatManipulatorDefault.php new file mode 100644 index 0000000..a0bdcba --- /dev/null +++ b/src/FormatManipulatorDefault.php @@ -0,0 +1,51 @@ +extractHeader($content); + } + + // If current position is at the end of the data set, extract the footer + // as well. + if ($current_position < $total_items) { + $this->extractFooter($content); + } + + // Write content to the output file. + return file_put_contents($output_file, $content, FILE_APPEND); + } + +} diff --git a/src/FormatManipulatorInterface.php b/src/FormatManipulatorInterface.php new file mode 100644 index 0000000..eea7e3e --- /dev/null +++ b/src/FormatManipulatorInterface.php @@ -0,0 +1,29 @@ +alterInfo('format_manipulator_info'); + $this->setCacheBackend($cache_backend, 'format_manipulator_plugins'); + } + + /** + * Creates a format manipulator plugin instance. + * + * @param string $plugin_id + * The plugin id. + * @param array $configuration + * Optional configuration for the plugin. + * + * @return \Drupal\views_data_export\FormatManipulatorInterface + * The plugin instantiated. + */ + public function createInstance($plugin_id, array $configuration = []) { + try { + $plugin_definition = $this->getDefinition($plugin_id); + } + catch (PluginNotFoundException $e) { + // Notify users about the absence of requested file format manipulator + // plugin. + throw new \Exception(dt('No format handler has been found for the format \'%plugin_id\'.', [ + '%plugin_id' => $plugin_id, + ])); + } + + // Apply the custom configuration. + array_merge($plugin_definition, $configuration); + + // Resolve plugin class. + $plugin_class = DefaultFactory::getPluginClass($plugin_id, $plugin_definition); + + return new $plugin_class(); + } + +} diff --git a/src/Plugin/FormatManipulator/FormatManipulatorCsv.php b/src/Plugin/FormatManipulator/FormatManipulatorCsv.php new file mode 100644 index 0000000..0598ed7 --- /dev/null +++ b/src/Plugin/FormatManipulator/FormatManipulatorCsv.php @@ -0,0 +1,23 @@ +realpath($context['sandbox']['vde_file']); + $previousExcel = \PHPExcel_IOFactory::load($vdeFileRealPath); + file_put_contents($vdeFileRealPath, $string); + $currentExcel = \PHPExcel_IOFactory::load($vdeFileRealPath); + + // Append all rows to previous created excel. + $rowIndex = $previousExcel->getActiveSheet()->getHighestRow(); + foreach ($currentExcel->getActiveSheet()->getRowIterator() as $row) { + if ($row->getRowIndex() == 1) { + // Skip header. + continue; + } + $rowIndex++; + $colIndex = 0; + foreach ($row->getCellIterator() as $cell) { + $previousExcel->getActiveSheet()->setCellValueByColumnAndRow($colIndex++, $rowIndex, $cell->getValue()); + } + } + + $objWriter = new \PHPExcel_Writer_Excel2007($previousExcel); + $objWriter->save($vdeFileRealPath); + */ + + throw new \Exception(dt('Xlsx format manipulation is not supported yet.')); + } + +} diff --git a/src/Plugin/FormatManipulator/FormatManipulatorXml.php b/src/Plugin/FormatManipulator/FormatManipulatorXml.php new file mode 100644 index 0000000..d6306ea --- /dev/null +++ b/src/Plugin/FormatManipulator/FormatManipulatorXml.php @@ -0,0 +1,35 @@ +)', '', $content); + + // Remove response root. + $content = str_replace('', '', $content); + } + + /** + * {@inheritdoc} + */ + protected function extractFooter(&$content) { + // Remove response header. + $content = str_replace('', '', $content); + } + +} diff --git a/src/Plugin/QueueWorker/FileWriter.php b/src/Plugin/QueueWorker/FileWriter.php new file mode 100644 index 0000000..c1dc738 --- /dev/null +++ b/src/Plugin/QueueWorker/FileWriter.php @@ -0,0 +1,14 @@ +query for future use + // into $this->renderViewChunk. + $view->build(); + + $render = $this->renderViewChunk($view, $export_count, $items_per_batch); + + // Write results to file. + $formatManipulator->handle($output_file, $render, $export_count + $items_per_batch, $export_items); + + // Release the cache. + $this->entityCacheClear($view); + } + + /** + * Performs view chunk rendering. + * + * @param \Drupal\views\ViewExecutable $view + * A view the result of which must be rendered. + * @param int $start_index + * Start query index. + * @param int $items_count + * Export items count. + * + * @return string + * Rendered data export display markup. + */ + private function renderViewChunk(ViewExecutable &$view, $start_index, $items_count) { + // Set query offset. + $view->setItemsPerPage($items_count); + $view->setOffset($start_index); + $view->query->setOffset($start_index); + $view->query->setLimit($items_count); + + // Force the view to be executed. + $view->executed = FALSE; + $view->build(); + + // Render the view output. + $view_render = $view->render(); + + return (string) $view_render['#markup']; + } + + /** + * Performs views related cached entity references abolition. + * + * @param \Drupal\views\ViewExecutable $view + * A view object cache of which must be cleared out. + */ + private function entityCacheClear(ViewExecutable &$view) { + // Release view result references. + $view->result = []; + } + +} diff --git a/tests/src/Functional/ViewsDataExportBatchTest.php b/tests/src/Functional/ViewsDataExportBatchTest.php index 6a6a4c4..15a6fa1 100644 --- a/tests/src/Functional/ViewsDataExportBatchTest.php +++ b/tests/src/Functional/ViewsDataExportBatchTest.php @@ -102,7 +102,7 @@ class ViewsDataExportBatchTest extends ViewTestBase { $path_to_file = parse_url($path_to_file, PHP_URL_PATH); $path_to_file = str_replace($_SERVER['REQUEST_URI'] . 'system/files', 'private:/', $path_to_file); $res3 = $this->readCsv(file_get_contents($path_to_file)); - $this->assertEquals(3, count($res3), 'Count of exported nodes is wrong.'); + $this->assertEquals(3, count($res3) - 1, 'Count of exported nodes is wrong.'); // Testing search api index's view. $this->indexItems('database_search_index'); @@ -113,7 +113,7 @@ class ViewsDataExportBatchTest extends ViewTestBase { $path_to_file = parse_url($path_to_file, PHP_URL_PATH); $path_to_file = str_replace($_SERVER['REQUEST_URI'] . 'system/files', 'private:/', $path_to_file); $res4 = $this->readCsv(file_get_contents($path_to_file)); - $this->assertEquals(8, count($res4), 'Count of exported test entities is wrong.'); + $this->assertEquals(8, count($res4)- 1, 'Count of exported test entities is wrong.'); } /** diff --git a/views_data_export.services.yml b/views_data_export.services.yml new file mode 100644 index 0000000..c1c4d26 --- /dev/null +++ b/views_data_export.services.yml @@ -0,0 +1,4 @@ +services: + plugin.manager.format_manipulator: + class: Drupal\views_data_export\FormatManipulatorLoader + parent: default_plugin_manager -- 2.25.1