<?php 
// $Id: node_import.module,v 1.50.2.3.2.1 2007/05/03 07:06:23 robrechtj Exp $

/**
 * @file
 * This modules provides a wizard at "administer >> content >> import"
 * to import a CSV file with nodes.
 */

/**
 * Implementation of hook_help().
 */
function node_import_help($section) {
  switch ($section) {
    case 'admin/help#node_import':
      $output = '<p>'. t('The node import module enables importing of nodes of any type into your site using comma separated values format (CSV) or tab separated values format (TSV). One possible use is with contact manager to import lists of contacts. Users want to be able to import content from other systems into their site.') .'</p>';
      $output .= '<p>'. t('Node import accepts a CSV or TSV file as input. CSV or TSV files can be generated using spreadsheet programs. Your CSV or TSV file must contain field names in its first row. These field names can be anything; except when using <a href="%external-http-drupal-org-node-24614">raw data</a> import type. Modules, such as contact_manager, will add additional import types.', array('%external-http-drupal-org-node-24614' => 'http://drupal.org/node/24614')) .'</p>';
      $output .= t('<p>You can</p>
<ul>
<li>read about the function that reads CSV files called <a href="%external-http-us3-php-net-fgetcvs" title="fgetcsv programmer function page">fgetcsv</a>.</li>
<li>import nodes at <a href="%admin-node-node_import">administer &gt;&gt; content &gt;&gt; import</a>.</li>
<li>administer node permissions at <a href="%admin-access-permission">administer &gt;&gt; access &gt;&gt; permissions &gt;&gt; node import</a>.</li>
</ul>', array('%external-http-us3-php-net-fgetcvs' => 'http://us3.php.net/fgetcsv', '%admin-node-node_import' => url('admin/node/node_import'), '%admin-access-permission' => url('admin/access/permission')));
      $output .= '<p>'. t('For more information please read the configuration and customization handbook <a href="%node_import">Node import page</a>.', array('%node_import' => 'http://www.drupal.org/handbook/modules/node_import/')) .'</p>';
      return $output;
  }
}

/**
 * Implementation of hook_menu().
 */
function node_import_menu($may_cache) {
  $links = array();
  if ($may_cache) {
    $links[] = array(
      'path' => 'admin/content/node_import',
      'title' => t('Import content'),
      'description' => t('Import nodes from a CSV or TSV file.'),
      'callback' => 'node_import_page',
      'access' => user_access('import nodes'),
    );
  }
  return $links;
}

/**
 * Implementation of hook_perm().
 */
function node_import_perm() {
  return array('import nodes');  
}

/**
 * Menu callback function.
 */
function node_import_page() {
  // Include the API.
  include_once(drupal_get_path('module', 'node_import') . '/node_import.api.inc');

  // Load hooks for supported modules.
  node_import_load_supported();

  $edit = array_merge((array)$_SESSION['node_import'], (array)$_POST);

  // This prevents drupal_get_form() from performing extra validation.
  unset($_POST);

  // validate the form
  if ($_SESSION['node_import_page'] && $_POST) {
    $function = $_SESSION['node_import_page'] .'_validate';
    $function($_POST['op'], $edit);
  }
  else {
    $_SESSION['node_import_page'] = '_node_import_start';
  }

  // Create the new page.
  $output = drupal_get_form($_SESSION['node_import_page'], $edit);

  // Save everything back to the session.
  $_SESSION['node_import'] = $edit;

  return $output;
}

/************************************************************
 * Node import wizard page 1: file and node content type.
 ************************************************************/

function _node_import_start(&$edit) {
  if ($edit['file']) {
    $form['file'] = array(
      '#type' => 'value',
      '#value' => $edit['file'],
    );
    $form[] = array(
      '#type' => 'item',
      '#title' => t('File'),
      '#value' => $edit['filename'] .' ('. format_size($edit['file']->filesize) .')',
    );
  }
  else {
    $form['file'] = array(
      '#type' => 'file',
      '#title' => t('Upload file'),
      '#size' => 48,
      '#description' => t('File containing the data to be imported.'),
    );
  }
  if ($edit['file_format'] && $edit['file_format'] != '') {
    $file_formats = _node_import_get_file_formats();
    $form[] = array(
      '#type' => 'item',
      '#title' => t('File format'),
      '#value' => $file_formats[$edit['file_format']],
    );
  }
  else {
    $form['file_format'] = array(
      '#type' => 'select',
      '#title' => t('File format'),
      '#options' => _node_import_get_file_formats(),
      '#default_value' => isset($edit['file_format']) ? $edit['file_format'] : '',
    );
  }
  
  // Only allow import of nodes for which the user has 'create' privileges
  $types = node_import_types(TRUE);
  $types['node_import'] = t('raw data (advanced)');
  $form['type'] = array(
    '#type' => 'select',
    '#title' => t('Type'),
    '#default_value' => $edit['type'],
    '#options' => $types,
  );

  if ($edit['file']) {
    $form[] = array(
      '#type' => 'submit',
      '#value' => t('Use a different file')
    );
  }
  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Next (mapping)'),
  );
  $form['#attributes'] = array('enctype' => 'multipart/form-data');
  return $form;
}

function _node_import_start_validate($op, &$edit) {
  global $user;
  global $base_url;

  // Delete an existing file if needed.
  if ($edit['op'] == t('Use a different file')) {
    file_delete($edit['file']->filepath);
    unset($edit['file']);
    unset($_SESSION['node_import']['file']);
    unset($edit['filename']);
    unset($_SESSION['node_import']['filename']);
    unset($edit['file_format']);
    unset($_SESSION['node_import']['file_format']);
    unset($edit['errors']);
    unset($_SESSION['node_import']['errors']);
  }
  else if ($edit['op'] == t('Next (mapping)')) {
    // If there is an uploaded file, save it to
    // drupal.node_import.{site_url}.{uid} in the temporary directory.
    $file = file_save_upload('file');
    if ($file) {
      $edit['filename'] = $file->filename;
      file_move($file, 'drupal.node_import.'. strtr($base_url, array('http://' => '', '/' => '.')) .'.'. $user->uid, 1);
      $edit['file'] = $file;
    }
    if (!$edit['file']) {
      form_set_error('file', t('You must select a file to import.'));
      return;
    }

    // Autodetect the fileformat if needed.
    if ($edit['file_format'] == '') {
      $format = _node_import_autodetect_file_format($edit['file']->filepath);
      if ($format == '') {
        form_set_error('file_format', t('Could not detect the file format.'));
        return;
      }
      $edit['file_format'] = $format;
    }

    $formats = _node_import_get_file_formats();
    if (!isset($formats[$edit['file_format']])) {
      form_set_error('file_format', t('You need to select a format from the list.'));
    }

    if (!$edit['type']) {
      form_set_error('type', t('You must select a content type.'));
      return;
    }

    if ($edit['type'] == 'node_import') {
      $_SESSION['node_import_page'] = '_node_import_preview';
    }
    else {
      $_SESSION['node_import_page'] = '_node_import_mapping';
    }
  }
}

/************************************************************
 * Node import wizard page 2: column/field mapping.
 ************************************************************/

function _node_import_mapping(&$edit) {
  $form = array();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File'),
    '#value' => $edit['filename'] .' ('. format_size($edit['file']->filesize) .') ',
  );
  $file_formats = _node_import_get_file_formats();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File format'),
    '#value' => $file_formats[$edit['file_format']],
  );
  $form[] = array(
    '#type' => 'item',
    '#title' => t('Type'),
    '#value' => node_get_types('name', $edit['type']),
  );

  if ($edit['type'] != 'node_import') {
    $fields = array_merge(array('' => t('<none>')), node_import_fields($edit['type']));

    $function = $edit['file_format'];
    $headers = $function($edit['file']->filepath, TRUE);

    if (!$edit['match']) {
      if ($savedmatch = node_import_automap($edit['type'], $headers)) {
        $edit['match'] = $savedmatch;
      }
    }

    $j = 0;
    while (($row = $function($edit['file']->filepath)) && $j++ < 5) {
      foreach ($row as $i => $value) {
        $datatmp[$i][] = check_plain($value);
      }
    }

    $form['match'] = array(
      '#tree' => TRUE,
      '#title' => t('Field mapping'),
      '#type' => 'fieldset',
      '#theme' => 'node_import_mapping_table',
    );

    foreach ($headers as $i => $value) {
      $form['match'][$i] = array(
        '#type' => 'select',
        '#title' => $value,
        '#default_value' => $edit['match'][$i],
        '#options' => $fields,
      );
    }
    
    foreach ($datatmp as $i => $value) {
      $form['match'][$i]['#description'] = '<ul><li>'. implode('</li><li>', $value) .'</li></ul>';
    }
  }

  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Back'),
  );
  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Next (options)'),
  );

  return $form;
}

function _node_import_mapping_validate($op, &$edit) {
  if ($edit['op'] == t('Back')) {
    $_SESSION['node_import_page'] = '_node_import_start';
  }
  else if ($edit['op'] == t('Next (options)')) {
    $_SESSION['node_import_page'] = '_node_import_options';
  }
}

/************************************************************
 * Node import wizard page 3: global options.
 ************************************************************/

function _node_import_options(&$edit) {
  $form = array();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File'),
    '#value' => $edit['filename'] .' ('. format_size($edit['file']->filesize) .') ',
  );
  $file_formats = _node_import_get_file_formats();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File format'),
    '#value' => $file_formats[$edit['file_format']],
  );
  $form[] = array(
    '#type' => 'item',
    '#title' => t('Type'),
    '#value' => node_get_types('name', $edit['type']),
  );

  if ($edit['type'] != 'node_import') {
    // load the previously filled in values of the global fields
    $global_values = array_merge((array)$_SESSION['node_import'], (array)$_POST);
    $global_values = isset($global_values['global']) ? $global_values['global'] : array();

    if ($global = module_invoke_all('node_import_global', $edit['type'], $global_values)) {
      $form['global'] = $global;
      $form['global']['#tree'] = TRUE;
    }
    else {
      $form['global'] = array(
        '#type' => 'markup',
        '#value' => t('<p>There are no global options you can set.</p>'),
      );
    }
  }

  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Back'),
  );
  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Next (preview)'),
  );

  return $form;
}

function _node_import_options_validate($op, &$edit) {
  if ($edit['op'] == t('Back')) {
    $_SESSION['node_import_page'] = '_node_import_mapping';
  }
  else if ($edit['op'] == t('Next (preview)')) {
    $_SESSION['node_import_page'] = '_node_import_preview';
  }
}

/************************************************************
 * Node import wizard page 4: preview.
 ************************************************************/

function _node_import_preview(&$edit) {
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File'),
    '#value' => $edit['filename'] .' ('. format_size($edit['file']->filesize) .') ',
  );
  $file_formats = _node_import_get_file_formats();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File format'),
    '#value' => $file_formats[$edit['file_format']],
  );
  $form[] = array(
    '#type' => 'item',
    '#title' => t('Type'),
    '#value' => node_get_types('name', $edit['type']),
  );
  $form[] = array(
    '#type' => 'markup',
    '#value' => '<p>'. t('Importing may take awhile, do not click "Apply (import nodes)" more than once. To see progress, look at <a href="%admin-content" target="_new">the "administer &gt;&gt; content" page in a new window</a>.', array('%admin-content' => url('admin/node'))) .'</p>',
  );

  if (!$edit['preview_count']) {
    $edit['preview_count'] = 5;
  }

  $edit['errors'] = array();
  $form[] = array(
    '#type' => 'item',
    '#value' => _node_import_get_nodes($edit['file']->filepath, $edit['type'], $edit['type'] == 'node_import' ? NULL : $edit['match'], $edit['global'], $edit['preview_count'], $edit['errors'], $edit['file_format']),
  );

  $form['preview_count'] = array(
    '#type' => 'select',
    '#title' => t('Number of entries to preview'),
    '#default_value' => $edit['preview_count'],
    '#options' => drupal_map_assoc(array(5, 10, 15, 25, 50, 100, 150, 200)),
  );

  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Back'),
  );
  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Reload'),
  );
  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Apply (import nodes)'),
  );

  return $form;
}

function _node_import_preview_validate($op, &$edit) {
  if ($edit['op'] == t('Back')) {
    if ($edit['type'] == 'node_import') {
      $_SESSION['node_import_page'] = '_node_import_start';
    }
    else {
      $_SESSION['node_import_page'] = '_node_import_options';
    }
  }
  else if ($edit['op'] == t('Apply (import nodes)')) {
    $_SESSION['node_import_page'] = '_node_import_import';
  }
  else if ($edit['op'] == t('Reload')) {
    $_SESSION['node_import_page'] = '_node_import_preview';
  }
}

/************************************************************
 * Node import wizard page 5: import.
 ************************************************************/

function _node_import_import(&$edit) {
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File'),
    '#value' => $edit['filename'] .' ('. format_size($edit['file']->filesize) .') ',
  );
  $file_formats = _node_import_get_file_formats();
  $form[] = array(
    '#type' => 'item',
    '#title' => t('File format'),
    '#value' => $file_formats[$edit['file_format']],
  );
  $form[] = array(
    '#type' => 'item',
    '#title' => t('Type'),
    '#value' => node_get_types('name', $edit['type']),
  );

  $form[] = array(
    '#type' => 'submit',
    '#value' => t('Delete CSV file from server'),
  );

  $edit['errors'] = array();
  $output = _node_import_get_nodes($edit['file']->filepath, $edit['type'], $edit['type'] == 'node_import' ? NULL : $edit['match'], $edit['global'], 0, $edit['errors'], $edit['file_format']);

  if (count($edit['errors']) > 0) {
    $form[] = array(
      '#type' => 'submit',
      '#value' => t('Download rows with errors'),
    );
  }

  $form[] = array(
    '#type' => 'item',
    '#value' => $output,
  );

  return $form;
}

function _node_import_import_validate($op, &$edit) {
  if ($edit['op'] == t('Delete file from server')) {
    if (file_delete($edit['file']->filepath)) {
      drupal_set_message(t('Deleted the file from the server.'));
    }
    $edit = array();
    $_SESSION['node_import_page'] = '_node_import_start';
  }
  else if ($edit['op'] == t('Download rows with errors')) {
    $_SESSION['node_import_page'] = '_node_import_errors';
  }
}

/************************************************************
 * Node import wizard page 6: download rows with errors.
 ************************************************************/

function _node_import_errors($edit) {
  if (!isset($edit['errors']) || empty($edit['errors'])) {
    return;
  }
  switch ($edit['file_format']) {
    case '_node_import_csv_get_row':
      header('Content-type: text/comma-separated-values');
      header('Content-Disposition: attachment; filename="rejected-'. $edit['filename'] .'"');

      // As a CSV line may span multiple lines and the fputcsv()
      // function is only for php 5.0, and we need to handle
      // quotes inside cells, we need to write this ourselves.
      // Based on: http://www.php.net/manual/en/function.fputcsv.php

      $quote = '"';
      $delimiter = ',';

      foreach ($edit['errors'] as $row) {
        $str = '';
        $write_delimiter = FALSE;

        foreach ($row as $cell) {
          $cell = str_replace($quote, $quote.$quote, $cell);
          if ($write_delimiter) {
            $str .= $delimiter;
          }
          $str .= $quote . $cell . $quote;
          $write_delimiter = TRUE;
        }

        print $str . "\n";
      }
      break;

    case '_node_import_tsv_get_row':
      header('Content-Type: text/tab-separated-values');
      header('Content-Disposition: attachment; filename="rejected-'. $edit['filename'] . '"');

      // To be correct and pedantic about this format, we would
      // need to check the number of columns of each row as the
      // text/tab-separated-values format specifies that each
      // row should contain the same number of tabs. We won't do
      // that.
      // See: http://www.iana.org/assignments/media-types/text/tab-separated-values

      foreach ($edit['errors'] as $row) {
        print implode("\t", $row) . "\n";
      }
      break;
  }
}

function _node_import_errors_validate($op, &$edit) {
  return _node_import_import_validate($op, $edit);
}

/**
 * Import and create nodes.
 * 
 * @param string $filepath
 *   Full path to the file to import.
 * @param string $type
 *   Node content type.
 * @param array $match
 *   Array of mappings.
 * @param array $global
 *   Array of global values.
 * @param int $preview
 *   Number of entries to preview or 0 to do the import.
 * @param array $error_rows
 *   Returns a list of error rows.
 * @param string $get_row
 *   Function used to get one row.
 */
function _node_import_get_nodes($filepath, $type, $match, $global, $preview, &$error_rows, $get_row = '_node_import_csv_get_row') {
  drupal_add_css(drupal_get_path('module', 'node_import') .'/node_import.css');

  $header = $get_row($filepath, TRUE);

  if (!$match) {
    $match = $header;
  }
  if ($match && !$preview) {
    db_query("DELETE FROM {node_import_mappings} WHERE type = '%s' AND csv_headers = '%s'", $type, serialize($header));
    db_query("INSERT INTO {node_import_mappings} (type, csv_headers, mapping) VALUES ('%s', '%s', '%s')",
             $type, serialize($header), serialize($match));
  }

  $j = 0;
  $success = 0;

  while (($row = $get_row($filepath)) && ($j++ < $preview || $preview == 0)) {
    $errors = array();

    // Create an empty node with only static and global options.
    if ($type == 'node_import') {
      $node = (object)array();
    }
    else {
      $node = (object)array_merge(array('type' => $type),
                                  (array)module_invoke_all('node_import_static', $type),
                                  (array)$global);
    }

    // Assign the mapped fields to the $node.    
    foreach ($row as $i => $value) {
      if (!empty($match[$i])) {
        $fieldname = $match[$i];
        $node->$fieldname = $value;
      }
      if ($match[$i] == 'name' && $type == 'node_import') {
        if (($account = user_load(array('name' => $value))) ||
            (is_numeric($value) && ($account = user_load(array('uid' => $value))))) {
          $node->uid = $account->uid;
          $node->name = $account->name;
        }
        else {
          $errors[] = t('The username %name does not exist.', array('%name' => $node->name));
        }
      }
    }

    // Report an error if we don't have a type ("raw input").
    if (!isset($node->type)) {
      $errors[] = t('You need to specify the node content type.');
    }

    // Prepare the node for import. We could have written the following loop
    // as: module_invoke_all('node_import_prepare', $node, $preview > 0);
    // but unfortunately module_invoke_all passes all argumens by value.
    foreach (module_list() as $module_name) {
      $function = $module_name . '_node_import_prepare';
      if (function_exists($function)) {
        $errors = array_merge((array)$errors, (array)$function($node, $preview > 0));
      }
    }

    // We can't do the normal validation we've done in 4.6 because a lot
    // of validation is done in $forms now instead of in node_validate($node).
    // Even if we could do it, we still have the problem that we are unable
    // to reset the form errors (coz it's a static variable inside
    // form_get_errors()). The only way I can think of to do real validation
    // is:
    // - use the current $node to fill in some $form,
    // - create a callback to validate *one* node which would be called
    //   for each node, this way form_get_errors() will only apply to
    //   one node.
    // I may do it one day, but let's duplicate the validation code for
    // nodes in this module instead.
    //node_validate($node);

    // See if we are allowed to create this node. This is again only for
    // "raw input".
    if (!node_access('create', $node)) {
      $errors[] = t('You are not authorized to create a node of type %type.', array('%type' => node_get_types('name', $node)));
    }

    // Ok, we're done. Preview the node or save it (if no errors).
    if (count($errors)) {
      $output .= '<div class="node_import-error">';
      $output .= theme('item_list', $errors, t('Can not import this node. Check following errors:'));
      $output .= node_view($node);
      $output .= '</div>';
      $error_rows[] = $row;
    }
    else if ($preview) {
      $node->in_preview = TRUE;
      $output .= '<div class="node_import-success">';
      $output .= node_view($node);
      $output .= '</div>';
    }
    else {
      node_submit($node);
      node_save($node);
      $success++;
    }

    // Now we have either imported the node or not. Report it to interested
    // modules.
    foreach (module_list() as $module_name) {
      $function = $module_name . '_node_import_postprocess';
      if (function_exists($function)) {
        $output .= $function($node, $preview > 0, count($errors) > 0); 
      }
    }

    //clean up
    unset($node);
    unset($errors);
  }
  $get_row('');

  if (!$preview) {
    if (count($error_rows)) {
      drupal_set_message(t("!errors Click 'Download rows with errors' for a CSV file of the failed rows.", array('!errors' => format_plural(count($error_rows), t('There was 1 error.'), t('There were @count errors.')))));
      $error_rows = array_merge(array($header), $error_rows);
    }
    drupal_set_message(format_plural($success, t('Successfully imported 1 node.'), t('Successfully imported @count nodes.')));
  }
  
  return $output;
}

/**
 * Theme table of field mappings.
 */
function theme_node_import_mapping_table($form) {
  $data = array();
  foreach ($form as $id => $select) {
    if (is_numeric($id)) {
      $title = $select['#title'];
      unset($select['#title']);
      $samples = $select['#description'];
      unset($select['#description']);
      $row = array();
      $row[] = $title;
      $row[] = drupal_render($select);
      $row[] = $samples;
      $data[] = $row;
    }
  }
  $header = array(t('CSV header'), t('Import to field'), t('Sample data'));
  $output = theme('table', $header, $data);
  return $output;
}

/**
 * Check if a date is valid and return the correct
 * timestamp to use. Returns -1 if the date is not
 * considered valid.
 */
function node_import_valid_date($date) {
  //TODO: really check whether the date is valid!!
  if (empty($date)) {
    return -1;
  }
  if (is_numeric($date) && $date > -1) {
    return $date;
  }
  $time = strtotime($date);
  if ($time < 0 || !$time) {
    return -1;
  }
  return $time;
}

/************************************************************************
 * Support for different file formats.
 ************************************************************************/

/**
 * Return an option list of supported formats. This will always
 * include 'auto'.
 *
 * TODO: as fgetcsv is not very robust (or rather, very), we try that last.
 */
function _node_import_get_file_formats() {
  return array(
    '' => t('Autodetect'),
    '_node_import_tsv_get_row' => t('Tab Separated Values (TSV) text file'),
    '_node_import_csv_get_row' => t('Comma Separated Values (CSV) text file'),
  );
}

/**
 * Returns an autodetected file format or '' if none found.
 */
function _node_import_autodetect_file_format($filepath) {
  $format = '';
  foreach (_node_import_get_file_formats() as $function => $title) {
    if (function_exists($function)) {
      $row = $function($filepath, TRUE);
      if (count($row) > 1) {
        $format = $function;
        break;
      }
    }
  }
  return $format;
}

/**
 * Get one row from the CSV file and return it as an array of
 * columns/fields.
 */
function _node_import_csv_get_row($filepath, $reset = FALSE) {
  static $handle;
  $size = variable_get('node_import_csv_size', 10000);
  $separator = variable_get('node_import_csv_separator', ',');

  if ($filepath == '') {
    unset($handle);
    return FALSE;
  }

  if (!isset($handle) || $reset) {
    if (isset($handle)) {
      fclose($handle);
    }
    $handle = fopen($filepath, 'r');
  }

  return fgetcsv($handle, $size, $separator);
}

/**
 * Get one row from the TSV file and return it as an array of
 * columns/fields.
 */
function _node_import_tsv_get_row($filepath, $reset = FALSE) {
  static $lines;
  static $line_no;
  $separator = variable_get('node_import_tsv_separator', "\t");

  if ($filepath == '') {
    unset($lines);
    return FALSE;
  }

  if (!isset($lines) || $reset) {
    $line_no = 0;
    $lines = file($filepath);
  }

  while ($line_no < count($lines)) {
    $line = rtrim($lines[$line_no], "\n\r ");
    $line_no++;
    if ($line != '') {
      return explode($separator, $line);
    }
  }
  return FALSE;
}