Index: millennium.admin.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.admin.inc,v
retrieving revision 1.1.2.2
diff -u -r1.1.2.2 millennium.admin.inc
--- millennium.admin.inc	15 Oct 2009 19:36:58 -0000	1.1.2.2
+++ millennium.admin.inc	16 Oct 2009 21:54:32 -0000
@@ -123,7 +123,6 @@
  * Callback for drupal_get_form for general settings form
  */
 function millennium_admin_settings() {
-
   // Check to see if allow_url_fopen is available, if not throw an error
   // so that the administrator knows that this will not work without it
   if (!ini_get('allow_url_fopen')) {
@@ -307,53 +306,336 @@
  * Callback for drupal_get_form that shows admin options for manually queueing items.
  */
 function millennium_admin_queue() {
-  $form['priority'] = array(
+  $form['source'] = array(
     '#type' => 'radios',
-    '#title' => t('Priority'),
-    '#options' => array(0 => "Normal", 1 => "Urgent"),
-    '#default_value' => 0,
-    '#description' => t('"Urgent" items are processed before "Normal" ones.'),
+    '#title' => t("Items to import"),
+    '#options' => array('list' => t('Item number listing'), 'range' => t('Item number range')),
+    '#default_value' => 'list',
+    '#prefix' => '
+<script>
+$(document).ready(function () {
+  $("#range-fieldset").hide();
+  $("input[@name=\'source\']").change(
+    function() {
+      //alert("changed!");
+      if ($("input[@name=\'source\']:checked").val() == "list") {
+        $("#range-fieldset").hide();
+        $("#list-fieldset").show();
+      } else {
+        $("#range-fieldset").show();
+        $("#list-fieldset").hide();
+      }
+      $(this).blur();
+    }
+  ) });
+</script>',
+    );
+  $form['list'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Item number listing'),
+    '#attributes' =>  array('id' => 'list-fieldset'),
+    );
+  $form['list']['queue'] = array(
+    '#type' => 'textarea',
+    '#title' => t('Item record numbers to queue for import (one per line)'),
+    '#description' => t('Record numbers are written like this: i123456, i123456x. You can get item numbers from Millennium Client\'s "Create Lists". NOTE: Bibliographic record numbers are NOT allowed.'),
+    '#default_value' => '',
+  );
+  $form['range'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Record number range'),
+    '#attributes' =>  array('id' => 'range-fieldset'),
+    );
+  $form['range']["start"] = array(
+    '#type' => 'textfield',
+    '#default_value' => variable_get('millennium_range_form_start', '100000'),
+    '#size' => 7,
+    '#title' => t('Starting item number'),
+  );
+  $form['range']["end"] = array(
+    '#type' => 'textfield',
+    '#default_value' => variable_get('millennium_range_form_end', '100001'),
+    '#size' => 7,
+    '#title' => t('Ending item number'),
+  );
+  $form['options'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Advanced options'),
+    '#collapsible' => TRUE,
+    '#collapsed' => TRUE,
   );
-  $form['force_update'] = array(
+  $form['options']['force_update'] = array(
     '#type' => 'radios',
-    '#title' => t('Force reimport for existing records?'),
-    '#options' => array(0 => "No", 1 => "Yes, overwrite existing records with current Millennium record"),
+    '#title' => t('Action for existing nodes'),
+    '#options' => array(0 => t("Skip, leave untouched"), 1 => t("Update with current bibliographic data")),
+    '#default_value' => 1,
+  );
+    $form['options']['priority'] = array(
+    '#type' => 'radios',
+    '#title' => t('Cron priority for these items'),
+    '#options' => array(0 => t("Low"), 1 => t("High")),
     '#default_value' => 0,
+    '#description' => t('All high-priority items will be processed before any of the low-priority items.'),
   );
-  $form['queue'] = array(
-    '#type' => 'textarea',
-    '#title' => t('Item record numbers to queue for import (one per line)'),
-    '#description' => t('Record numbers are written like this: i123456, i123456x. You can get desired item numbers from Millennium Client\'s "Create Lists". Bibliographic record numbers are NOT permitted.'),
-    '#default_value' => '',
+  $form['submit']['using_cron'] = array(
+    '#type' => 'submit',
+    '#value' => t('Process these records during cron runs'),
   );
-  $form['submit'] = array(
+  $form['submit']['using_batch'] = array(
     '#type' => 'submit',
-    '#value' => t('Add above items to queue'),
+    '#value' => t('Process these records now'),
   );
   return $form;
 }
 
 /**
+ * Validate function for millennium_admin_queue() form
+ */
+function millennium_admin_queue_validate($form, &$form_state) {
+  if ($form_state["values"]["source"] == "range") {
+    foreach (array("start", "end") as $element) {
+      if (!is_int($form_state["values"][$element] + 0)) {
+        form_set_error($element, t("Must be a number"));
+      }
+      if ($form_state["values"][$element] + 0 <10000) {
+        form_set_error($element, t("Must be equal or higher than 10000"));
+      }
+    }
+    if ($form_state["values"]["end"] <= $form_state["values"]["start"]) {
+      form_set_error("end", t("Ending number must be higher than starting number."));
+    }
+  }
+
+  if ($form_state["values"]["source"] == "list") {
+    if (trim($form_state['values']['queue']) == "") {
+      form_set_error("queue", t("You must enter at least one item number in the list."));
+      return;
+    }
+    $lines = explode("\n", $form_state['values']['queue']);
+    $errors = array();
+    foreach ($lines as $line) {
+      $recnum = trim($line);
+      if ($recnum == "") {
+        continue;
+      }
+      if (!preg_match('/^i[0-9]+x*$/i', $recnum)) {
+        $errors[] = $line;
+      }
+    }
+    if ($errors) {
+      form_set_error("queue", t("These are not Millennium item record numbers:") . theme('item_list', $errors), "error");
+    }
+  }
+}
+
+/**
  * Submit function for millennium_admin_queue() form
  */
 function millennium_admin_queue_submit($form, &$form_state) {
-  $priority = $form_state['values']['priority'];
-  $force = $form_state['values']['force_update'];
-  $lines = explode("\n", $form_state['values']['queue']);
-  $queued = 0;
-  foreach ($lines as $line) {
-    $recnum = trim($line);
-    if (!preg_match('/^i[0-9]+x*$/i', $recnum)) {
-      drupal_set_message("Ignoring '$recnum': not a Millennium item record number", "error");
+  #dpm($form_state);
+  #return;
+
+  if ($form_state["submitted"] == true) {
+
+    // Store some values for later
+    if ($form_state["values"]["source"] == "range") {
+      variable_set('millennium_range_form_start', $form_state["values"]["start"]);
+      variable_set('millennium_range_form_end', $form_state["values"]["end"]);
+    }
+
+    if ($form_state["clicked_button"]["#value"] == t("Process these records during cron runs")) {
+
+      // When? Cron runs
+      if ($form_state["values"]["source"] == "list") {
+        // Import from list
+        $lines = explode("\n", $form_state['values']['queue']);
+        $queued = 0;
+        foreach ($lines as $line) {
+          $recnum = trim($line);
+          if (!preg_match('/^i[0-9]+x*$/i', $recnum)) {
+            drupal_set_message("Ignoring '$recnum': not a Millennium item record number", "error");
+          }
+          else {
+            $recnum = drupal_substr($recnum, 0, drupal_strlen($recnum)-1);
+            $ok = db_query("INSERT INTO {millennium_import_queue} (item_recnum, priority, force_update) VALUES ('%s', %d, %d)",
+              $recnum,
+              $form_state['values']['priority'],
+              $form_state['values']['force_update']);
+            if ($ok)
+              $queued++;
+          }
+        }
+      }
+
+      if ($form_state["values"]["source"] == "range") {
+        // Import from range
+        $start = $form_state["values"]["start"];
+        $end = $form_state["values"]["end"];
+        for ($num = $start; $num < $end; $num++) {
+          $ok = db_query("INSERT INTO {millennium_import_queue} (item_recnum, priority, force_update) VALUES ('%s', %d, %d)",
+            "i" . $num,
+            $form_state['values']['priority'],
+            $form_state['values']['force_update']);
+          if ($ok) {
+            $queued++;
+          }
+        }
+      }
+      drupal_set_message( t('@count items queued successfully. They will be processed during subsecuent !cron runs', array("@count" => $queued, '!cron' => l('cron', 'admin/reports/status/run-cron'))));
+
     }
     else {
-      $recnum = drupal_substr($recnum, 0, drupal_strlen($recnum)-1);
-      $ok = db_query("INSERT INTO {millennium_import_queue} (item_recnum, priority, force_update) VALUES ('%s', %d, %d)",
-        $recnum,
-        $priority, $force);
-      if ($ok)
-        $queued++;
+      // When? Right now!
+      $item_recnums = array();
+
+      if ($form_state["values"]["source"] == "list") {
+        // Import from list
+        $lines = explode("\n", $form_state['values']['queue']);
+        $queued = 0;
+        foreach ($lines as $line) {
+          $recnum = trim($line);
+          if (!preg_match('/^i[0-9]+x*$/i', $recnum)) {
+            drupal_set_message("Ignoring '$recnum': not a Millennium item record number", "error");
+          }
+          else {
+            $recnum = drupal_substr($recnum, 0, drupal_strlen($recnum)-1);
+            $item_recnums[$recnum] = $recnum;
+          }
+        }
+      }
+
+      if ($form_state["values"]["source"] == "range") {
+        // Import from range
+        $start = $form_state["values"]["start"];
+        $end = $form_state["values"]["end"];
+        for ($num = $start; $num < $end; $num++) {
+          $item_recnums["i{$num}"] = "i{$num}";
+        }
+      }
+
+      if (sizeof($item_recnums) > 0) {
+        millennium_batch_import($item_recnums, $form_state['values']['force_update']);
+       $form["#redirect"] = "admin/settings/millennium/batch_import";
+      }
     }
   }
-  drupal_set_message( t('@count items queued successfully', array("@count" => $queued) ));
+
+}
+
+/**
+ * Batch API implementation
+ */
+function millennium_batch_import($item_recnums, $force_update = true) {
+  $batch = array(
+    'operations' => array(
+      array('millennium_batch_import_process', array($item_recnums, $force_update)),
+    ),
+    'finished' => 'millennium_batch_import_finished',
+    'title' => t('Importing items from Millennium site @site', array('@site' => millennium_get_real_baseurl())),
+    'init_message' => t('Batch import is starting.'),
+    'file' => drupal_get_path('module', 'millennium') . '/millennium.admin.inc',
+    //'progress_message' => t('Reindexed @current out of @total.'),
+    'error_message' => t('Batch importing has encountered an error.'),
+  );
+  batch_set($batch);
+}
+
+/**
+* Batch Operation Callback
+*/
+function millennium_batch_import_process($item_recnums, $force_update, &$context) {
+  timer_start("millennium_batch_import_process");
+  require_once(drupal_get_path("module", "millennium") . "/millennium.import.inc");
+
+  // We can safely process this limit without a timeout.
+  $limit = variable_get('millennium_webopac_maxrecords', 50);
+  $chunks = array_chunk($item_recnums, $limit);
+
+  if (!isset($context['sandbox']['progress'])) {
+    $context['sandbox']['progress'] = 0;
+    $context['sandbox']['current_item'] = 0;
+    $context['sandbox']['tot_items'] = sizeof($item_recnums);
+    $context['sandbox']['tot_attempted'] = 0;
+    $context['sandbox']['tot_imported'] = 0;
+    $context['sandbox']['tot_notfound'] = 0;
+    $context['sandbox']['tot_fail'] = 0;
+    $context['sandbox']['elapsed'] = 0;
+    $context['sandbox']['max'] = sizeof($chunks);
+  }
+
+  // Here we actually perform our processing on the current chunk.
+  $chunk = $chunks[$context['sandbox']['current_item']];
+  $context['sandbox']['tot_attempted'] += sizeof($chunk);
+  $fetched = millennium_mass_fetch($chunk);
+  $context['sandbox']['tot_notfound'] += sizeof($fetched['not_found']);
+  // Import successful fetches only
+  foreach ($fetched['found'] as $data) {
+    $result = millennium_import_update_item(
+      $data['item_recnum'],
+      $force_update,
+      $data['marc'],
+      $data['bib_recnum']
+    );
+
+    if ($result["success"] !== false) {
+      $context['sandbox']['tot_imported']++;
+      #$context['results'][] = l(check_plain($result["node"]->title), "node/" . $result["node"]->nid);
+    } else {
+      $context['sandbox']['tot_fail']++;
+      /*
+       watchdog("Millennium",
+        "Queue #@id: Failed to import node: item #@recnum, error: @error",
+        array("@id" => $ids[$data['item_recnum']], "@recnum" => $data['item_recnum'], "@error" => $result["error"])
+      );
+      */
+    }
+  }
+
+  // Update our progress information.
+  $context['sandbox']['elapsed'] += (timer_read("millennium_batch_import_process")/1000);
+  #timer_stop("millennium_batch_import_process");
+  $context['sandbox']['progress']++;
+  $context['sandbox']['current_item']++;
+  $context['message'] = t("<ul><li>@imported imported<li>@notfound not found on WebOpac<li>@failed could not import<li>@pending pending<li>@items items per second</ul>",
+    array(
+      "@notfound" => $context['sandbox']['tot_notfound'],
+      "@imported" => $context['sandbox']['tot_imported'],
+      "@failed" => $context['sandbox']['tot_fail'] ,
+      "@pending" => $context['sandbox']['tot_items'] - $context['sandbox']['tot_attempted'],
+      "@items" => sprintf("%2.1f", $context['sandbox']['tot_attempted'] / $context['sandbox']['elapsed'])
+      )
+  );
+
+  // Update results data for millennium_batch_import_finished()
+  $context['results'] = array();
+  foreach (array('elapsed', 'tot_items','tot_attempted', 'tot_imported', 'tot_notfound', 'tot_fail') as $index) {
+    $context['results'][$index] = $context['sandbox'][$index];
+  }
+  $context['results']['message'] = $context['message'];
+
+  // Inform the batch engine that we are not finished,
+  // and provide an estimation of the completion level we reached.
+  if ($context['sandbox']['progress'] != $context['sandbox']['max']) {
+    $context['finished'] = $context['sandbox']['progress'] / $context['sandbox']['max'];
+  }
+}
+
+/**
+* Batch 'finished' callback
+*/
+function millennium_batch_import_finished($success, $results, $operations) {
+  if ($success) {
+    // Here we do something meaningful with the results.
+    #$message = format_plural(count($results), '1 item successfully processed.', '@count items successfully imported.');
+    #$message .= theme('item_list', $results);
+    $message = $results['message'];
+  }
+  else {
+    // An error occurred.
+    // $operations contains the operations that remained unprocessed.
+    $error_operation = reset($operations);
+    $message = t('An error occurred while processing @num with arguments :', array('@num' => $error_operation[0])) . print_r($error_operation[0], TRUE);
+  }
+  millennium_time_history($results['elapsed'], $results['tot_attempted'], $results['tot_notfound'], $results['tot_imported'], $results['tot_fail']);
+  drupal_set_message($message, 'error');
 }
Index: millennium.cron.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.cron.inc,v
retrieving revision 1.1.2.2
diff -u -r1.1.2.2 millennium.cron.inc
--- millennium.cron.inc	15 Oct 2009 19:36:59 -0000	1.1.2.2
+++ millennium.cron.inc	16 Oct 2009 18:37:05 -0000
@@ -61,8 +61,8 @@
       db_query("DELETE FROM {millennium_import_queue} where id=%d", $data->id);
     }
     $tot_attempted += sizeof($item_recnums);
-    // Chunk array into groups of 25 (maximum number allowed by bookcart)
-    $item_recnums_chunks = array_chunk($item_recnums, 25);
+    // Chunk array into groups of 50 (maximum number allowed by bookcart)
+    $item_recnums_chunks = array_chunk($item_recnums, 50);
     foreach ($item_recnums_chunks as $item_recnums_chunk) {
       $fetched = millennium_fetch_records_via_bookcart($item_recnums_chunk);
 
@@ -135,8 +135,8 @@
   }
   $tot_attempted += sizeof($item_recnums);
 
-  // Chunk array into groups of 25 (maximum number allowed by bookcart)
-  $item_recnums_chunks = array_chunk($item_recnums, 25);
+  // Chunk array into groups of 50 (maximum number allowed by bookcart)
+  $item_recnums_chunks = array_chunk($item_recnums, 50);
   foreach ($item_recnums_chunks as $item_recnums_chunk) {
     $fetched = millennium_fetch_records_via_bookcart($item_recnums_chunk);
     #dpm($fetched);
@@ -209,17 +209,3 @@
   );
 }
 
-/**
- * Store times for latest MILLENNIUM_PERFORMANCE_HISTORY_SIZE cron runs
- */
-function millennium_time_history($elapsed_time, $tot_attempted, $tot_notfound, $tot_imported, $tot_fail) {
-  $time_history = variable_get("millennium_time_history", array());
-  if (!is_array($time_history)) {
-    $time_history = array();
-  }
-  if (sizeof($time_history) >= MILLENNIUM_PERFORMANCE_HISTORY_SIZE) {
-    array_shift($time_history);
-  }
-  $time_history[] = array("timestamp" => time(), "items" => $tot_attempted, "not_found" => $tot_notfound, "imported" => $tot_imported, "time" => $elapsed_time);
-  variable_set("millennium_time_history", $time_history);
-}
Index: millennium.module
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/millennium.module,v
retrieving revision 1.13.2.33.2.2.2.29
diff -u -r1.13.2.33.2.2.2.29 millennium.module
--- millennium.module	15 Oct 2009 20:14:12 -0000	1.13.2.33.2.2.2.29
+++ millennium.module	16 Oct 2009 21:52:13 -0000
@@ -117,8 +117,8 @@
     'weight' => -10,
   );
 
-  $items[MILLENNIUM_SETTINGS_PATH .'/queue_add'] = array(
-    'title' => 'Add items to import queue',
+  $items[MILLENNIUM_SETTINGS_PATH .'/manual-import'] = array(
+    'title' => 'Manual import',
     'description' => 'Add item record numbers to import during subsequent cron calls.',
     'page callback' => 'drupal_get_form',
     'page arguments' => array('millennium_admin_queue'),
@@ -419,7 +419,7 @@
     // Add taxonomy to new node
     #millennium_add_taxonomy_to_node($nodeobject, $marc_parsed);
 
-    return array("success" => true);
+    return array("success" => true, "node" => $nodeobject);
   }
   else {
     // There is an existing node for this item's parent bibrecord
@@ -477,7 +477,7 @@
       // Update
       millennium_node_update($nodeobject, $item_recnum, $bib_recnum);
 
-      return array("success" => true);
+      return array("success" => true, "node" => $nodeobject);
     } else {
       // TODO: Use Success?
       return array("success" => true, "error" => "Item belongs to existing bib item ". $bib_import_history->bib_recnum ." in node ". $bib_import_history->nid);
@@ -2020,13 +2020,26 @@
  * Returns an array of arrays with holding information for a bib record
  * @param recnum Bib Record number without .b and check digit (example: '10000')
  */
-function millennium_get_holdings_info($recnum) {
+function millennium_get_holdings_info($recnum, $fetched_html = null) {
+  static $cache;
+
   $items = array();
-  $result = millennium_fetch_recordpage($recnum, "items");
-  if (! $result->data) {
-    return false;
+  if ($fetched_html == null) {
+    // Try the static cache
+    if (isset($cache[$recnum])) {
+      return $cache[$recnum];
+    }
+    // Try Drupal's cache
+    $cid = "millennium_get_holdings_info-$recnum";
+    if ($cache[$recnum] = cache_get($cid)) {
+      return $cache[$recnum]->data;
+    }
+    $result = millennium_fetch_recordpage($recnum, "items");
+    if (! $result->data) {
+      return false;
+    }
+    $fetched_html = $result->data;
   }
-  $fetched_html = $result->data;
 
   static $code_2_human = array(
     '1' => 'location',
@@ -2107,6 +2120,8 @@
       $items[] = $field;
     }
   }
+  $cache[$recnum] = $items;
+  cache_set($cid, $items, 'cache', time()+(24*3600)); // Cache for one day
   return $items;
 }
 
@@ -2888,3 +2903,18 @@
   }
   return $requirements;
 }
+
+/**
+ * Store times for latest MILLENNIUM_PERFORMANCE_HISTORY_SIZE cron runs
+ */
+function millennium_time_history($elapsed_time, $tot_attempted, $tot_notfound, $tot_imported, $tot_fail) {
+  $time_history = variable_get("millennium_time_history", array());
+  if (!is_array($time_history)) {
+    $time_history = array();
+  }
+  if (sizeof($time_history) >= MILLENNIUM_PERFORMANCE_HISTORY_SIZE) {
+    array_shift($time_history);
+  }
+  $time_history[] = array("timestamp" => time(), "items" => $tot_attempted, "not_found" => $tot_notfound, "imported" => $tot_imported, "time" => $elapsed_time);
+  variable_set("millennium_time_history", $time_history);
+}