Index: millennium.cron.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.cron.inc,v
retrieving revision 1.1.2.1
diff -u -r1.1.2.1 millennium.cron.inc
--- millennium.cron.inc	13 Oct 2009 19:49:45 -0000	1.1.2.1
+++ millennium.cron.inc	13 Oct 2009 22:10:39 -0000
@@ -2,6 +2,7 @@
 //$Id: millennium.cron.inc,v 1.1.2.1 2009/10/13 19:49:45 janusman Exp $
 
 function millennium_cron_do() {
+  require_once(drupal_get_path("module", "millennium") . "/millennium.import.inc");
   $start_time = microtime(true);
 
   // Is URL set?
@@ -25,16 +26,16 @@
   $max_to_import = intval(variable_get("millennium_webopac_maxrecords", 50));
 
   // How many back-to-back "not found" records before quitting the crawl
-  $max_notfound = $max_to_import;
-  $tot_imported = 0;
+  #$max_notfound = $max_to_import;
   $sequential_tot_notfound = 0;
+  $tot_imported = 0;
   $tot_fail = 0;
-
   $will_auto_crawl = variable_get('millennium_crawl_flag', "0") != "0";
+
+  // MANUAL IMPORT QUEUE =======================================================
   if ($will_auto_crawl) {
     $max_to_import_from_queue = $max_to_import * 0.5;
-  }
-  else {
+  } else {
     $max_to_import_from_queue = $max_to_import;
   }
   /* Use the queue table first, to import up to $max_to_import items.
@@ -46,31 +47,56 @@
     1 => $max_to_import_from_queue * 0.8,
     0 => $max_to_import_from_queue,
   );
-
   foreach ($queue_jobs as $priority => $limit) {
+    // Get items to import from DB
     $handle = db_query("SELECT id, item_recnum, force_update FROM {millennium_import_queue} WHERE priority=%d", $priority);
     $stop = $tot_imported + $limit;
+    $item_recnums = array();
     while ($data = db_fetch_object($handle)) {
-      if ($tot_imported > $stop || $tot_imported > $max_to_import) break;
-      $result = millennium_import_update_item($data->item_recnum, $data->force_update);
-      // TODO: Handle existing item records that map to imported bib records (currently they return success = false?)
-      if ($result["success"] !== false) {
-        $tot_imported++;
-      }
-      else {
+      $item_recnums[] = $data->item_recnum;
+      $force_update[$data->item_recnum] = $data->force_update;
+      #$ids[$data->item_recnum] = $data->id;
+      db_query("DELETE FROM {millennium_import_queue} where id=%d", $data->id);
+    }
+    // Chunk array into groups of 25 (maximum number allowed by bookcart)
+    $item_recnums_chunks = array_chunk($item_recnums, 25);
+    foreach ($item_recnums_chunks as $item_recnums_chunk) {
+      $fetched = millennium_fetch_records_via_bookcart($item_recnums_chunk);
+
+      // Remove not-found items from queue
+      foreach ($fetched['not_found'] as $item_recnum) {
+        #db_query("DELETE FROM {millennium_import_queue} where id=%d", $ids[$item_recnum]);
         $tot_fail++;
-        /*
-        watchdog("Millennium",
-          "Queue #@id: item #@recnum import error: @error",
-          array("@id" => $data->id, "@recnum" => $data->item_recnum, "@error" => $result["error"])
+        $import_errors[] = array("recnum" => $item_recnum, "error" => 'Not found');
+      }
+
+      // Import successful fetches only
+      foreach ($fetched['found'] as $data) {
+        $result = millennium_import_update_item(
+          $data['item_recnum'],
+          $force_update[$data['item_recnum']],
+          $data['marc'],
+          $data['bib_recnum']
         );
-        */
 
-        // Build a list of errors
-        $import_errors[] = array("recnum" => $data->item_recnum, "error" => $result["error"]);
+        if ($result["success"] !== false) {
+          $tot_imported++;
+        } else {
+          $tot_fail++;
+          watchdog("Millennium",
+            "Queue #@id: Failed to import node: item #@recnum, error: @error",
+            array("@id" => $ids[$data['item_recnum']], "@recnum" => $data['item_recnum'], "@error" => $result["error"])
+          );
+          // Build a list of errors
+          $import_errors[] = array("recnum" => $data['item_recnum'], "error" => $result["error"]);
+        }
+
+        if ($tot_imported > $stop || $tot_imported > $max_to_import) {
+          break;
+        }
+
       }
-      db_query("DELETE FROM {millennium_import_queue} where id=%d", $data->id);
-    }
+    } // foreach ($item_recnums_chunks as $item_recnums_chunk)
   }
 
   // Send errors via email
@@ -96,28 +122,45 @@
     return;
   }
 
-  $rollover_limit = intval($default_end + ($default_end - $default_start)*.05);
 
-  // Pre-fetch pages
-  // TODO: CHANGE to the new method in recordtest.php
-  if (variable_get('millennium_webopac_pipeline_connections', 2)) {
-    require_once("millennium.import.inc");
-    $count = millennium_fetch_recordpage_preload($beginning_rec_num, $beginning_rec_num + $max_to_import, "plain");
-    if ($count !== false) {
-      // Change maximum number to import to just those that could be prefetched
-      $max_to_import = $count;
-    }
-  }
 
-  // TODO: Delete items that weren't found HERE (or after next loop?)
+  // AUTO CRAWL ================================================================
+  $rollover_limit = intval($default_end + ($default_end - $default_start)*.05);
+  $item_recnums = array();
+  for ($num = $beginning_rec_num; $num < $beginning_rec_num + $max_to_import; $num++) {
+    $item_recnums["i{$num}"] = "i{$num}";
+  }
 
-  while (true) { // TODO: CYCLE only on found items
-    $result = millennium_import_update_item("i". $current_rec_num); // TODO: ADD marc text, bib number here since we'll have it by now!
+  // Chunk array into groups of 25 (maximum number allowed by bookcart)
+  $item_recnums_chunks = array_chunk($item_recnums, 25);
+  foreach ($item_recnums_chunks as $item_recnums_chunk) {
+    $fetched = millennium_fetch_records_via_bookcart($item_recnums_chunk);
+    foreach ($fetched['found'] as $data) {
+      $result = millennium_import_update_item(
+        $data['item_recnum'],
+        $force_update[$data['item_recnum']],
+        $data['marc'],
+        $data['bib_recnum']
+      );
+      if ($result["success"] !== false) {
+        $tot_imported++;
+      } else {
+        $tot_fail++;
+        watchdog("Millennium",
+          "Crawl: Failed to import node: item #@recnum, error: @error",
+          array("@recnum" => $data['item_recnum'], "@error" => $result["error"])
+        );
+      }
+    }
 
-    if ($result["success"] !== false) {
-      $tot_imported++;
+    // If any records found, reset the back-to-back not-found record count
+    if (sizeof($fetched["found"] > 0 )) {
+      $sequential_tot_notfound = 0;
 
       //Store last successful bibrec imported
+      $tmp = array_pop($fetched['found']);
+      $current_rec_num = substr($tmp['item_recnum'], 1) + 0; // Remove the "i" from i100000
+      #drupal_set_message("current_rec_num = $current_rec_num");
       if ($current_rec_num > $last_succesful_rec_num) {
         variable_set('millennium_webopac_latest_successful_itemrecord', $current_rec_num);
         $last_succesful_rec_num = $current_rec_num;
@@ -127,49 +170,43 @@
       #if ($current_rec_num > $rollover_limit) {
       #  $rollover_limit = intval($current_rec_num + ($current_rec_num - $default_start)*.05);
       #}
+    }
 
-      // Reset the back-to-back not-found record count
-      $sequential_tot_notfound = 0;
+    #if ($tot_imported >= $max_to_import) break;
 
-      #if ($tot_imported >= $max_to_import) break;
-    }
-    else {
-      /*
-      watchdog("Millennium",
-        "Crawl: item #@recnum import error: @error",
-        array("@recnum" => "i" . $current_rec_num, "@error" => $result["error"])
-      );
-      */
-      $sequential_tot_notfound++;
-      $tot_fail++;
-      #if ($tot_notfound >= $max_notfound) break;
-
-      // Do rollover when no records were found inthis cron run after 5% the estimated database size
-      if ($current_rec_num > $rollover_limit && $sequential_tot_notfound >= $max_to_import) {
-        watchdog("Millennium", "Crawl will start over, as it has detected no records after #@recnum", array("@recnum" => $rollover_limit));
-        _millennium_crawl_restart();
-        break;
-      }
+    // Increase not-found counter by number of items not found
+    $sequential_tot_notfound += sizeof($results['not_found']);
 
-    }
-    $current_rec_num++;
-    variable_set('millennium_webopac_current_itemrecord', $current_rec_num);
+    // Store last tried item
+    $last_tried_itemnum = substr(array_pop($item_recnums_chunk), 1) + 0;
+    variable_set('millennium_webopac_current_itemrecord', $last_tried_itemnum);
+  }
 
-    // Adjust ending record if records are being found past that number
-    if ($current_rec_num > $default_end) {
+  // Do rollover when no records were found in this cron run after 5% the estimated database size
+  if ($last_tried_itemnum > $rollover_limit && $sequential_tot_notfound >= $max_to_import) {
+    watchdog("Millennium", "Crawl will start over, as it has detected no records after #@recnum", array("@recnum" => $rollover_limit));
+    _millennium_crawl_restart();
+    break;
+  }
+
+  // Adjust ending record if records are being found past that number
+  if ($current_rec_num > $default_end) {
       variable_set('millennium_webopac_end_itemrecord', $current_rec_num);
       $default_end = $current_rec_num;
-    }
-
-    // Stop when maximum number of items crawled
-    if ($current_rec_num == $beginning_rec_num + $max_to_import) {
-      break;
-    }
-
   }
+
   $elapsed_time = microtime(true) - $start_time;
+  millennium_time_history($elapsed_time, $tot_imported, $tot_fail);
 
-  // Store times for latest MILLENNIUM_PERFORMANCE_HISTORY_SIZE cron runs
+  watchdog("Millennium", "Cron import finished: @imported imported, @failed failed in @time seconds",
+    array("@imported" => $tot_imported, "@failed" => $tot_fail+0, "@time" => sprintf("%2.1f", $elapsed_time))
+  );
+}
+
+/**
+ * Store times for latest MILLENNIUM_PERFORMANCE_HISTORY_SIZE cron runs
+ */
+function millennium_time_history($elapsed_time, $tot_imported, $tot_fail) {
   $time_history = variable_get("millennium_time_history", array());
   if (!is_array($time_history)) {
     $time_history = array();
@@ -179,8 +216,4 @@
   }
   $time_history[] = array("timestamp" => time(), "items" => $tot_imported + $tot_fail, "time" => $elapsed_time);
   variable_set("millennium_time_history", $time_history);
-
-  watchdog("Millennium", "Cron import finished: @imported imported, @failed failed in @time seconds",
-    array("@imported" => $tot_imported, "@failed" => $tot_fail+0, "@time" => sprintf("%2.1f", $elapsed_time))
-  );
 }
Index: millennium.import.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.import.inc,v
retrieving revision 1.1.2.1
diff -u -r1.1.2.1 millennium.import.inc
--- millennium.import.inc	13 Oct 2009 19:49:45 -0000	1.1.2.1
+++ millennium.import.inc	13 Oct 2009 22:08:59 -0000
@@ -2,162 +2,147 @@
 // $Id: millennium.import.inc,v 1.1.2.1 2009/10/13 19:49:45 janusman Exp $
 
 /**
- * Preloads pages into the cache using simultaneous requests to the server (using user-defined limits)
- * @param start_item_recnum number of first item record number (e.g. 123456) to fetch
- * @param end_item_recnum number of first item record number (e.g. 123456) to fetch
- * @param mode type of record to fetch (plain, xml, item)
+ * Gets a sequential number of records obeying PHP's max_execution_time setting.
+ * @param array $item_recnums Array of item numbers to fetch.
  */
-function millennium_fetch_recordpage_preload($start_item_recnum, $end_item_recnum, $mode = "plain") {
-  global $millennium_fetch_recordpage_cache;
-  $start_time = microtime(true);
+function millennium_mass_fetch($item_recnums) {
+
   if (ini_get('max_execution_time')<20) {
     return false;
   }
+
   // Don't devote more than 75% of max_execution_time to prefetch
-  $max_time = time() + (ini_get('max_execution_time') * 0.75);
+  $max_time = ini_get('max_execution_time') * 0.75;
 
-  $max_connections = variable_get('millennium_webopac_pipeline_connections', 2);
-  $max_errors = 10;
-  $open_connections = 0;
-  $tot_fetched = 0;
-  $errors = 0;
-  $current_item_recnum = $start_item_recnum;
-
-  while (time() < $max_time && $errors < $max_errors) {
-
-    // Open maximum number of connections
-    while(time() < $max_time && $open_connections < $max_connections && $current_item_recnum <= $end_item_recnum) {
-      $url = millennium_permalink("i". $current_item_recnum, $mode);
-      #drupal_set_message("millennium_fetch_recordpage_preload(): Opening connection for $current_item_recnum. Currently open: $open_connections");
-      $fp = _millennium_nonblocking_http_request($url);
-      if (!$fp) {
-        $errors++;
-        if ($errors >= $max_errors) {
-          watchdog("Millennium", "Prefetch aborting, reached @count errors", array("@count" => $max_errors));
-          #drupal_set_message("millennium_fetch_recordpage_preload(): Maximum errors reached!");
-          break;
-        }
-      }
-      $connections[$current_item_recnum] = $fp;
-      $buffer[$current_item_recnum] = "";
-      $open_connections++;
-      $current_item_recnum++;
-    }
+  // Chunk array into groups of 25 (maximum number allowed by bookcart)
+  $chunks = array_chunk($item_recnums, 25);
+  #dpm($chunks);
+  $results = array('found' => array(), 'not_found' => array());
+  foreach ($chunks as $chunk) {
+    $result = millennium_fetch_records_via_bookcart($chunk);
+    $results['found'] = array_merge($results['found'], $result['found']);
+    $results['not_found'] = array_merge($results['not_found'], $result['not_found']);
 
-    if ($open_connections == 0 && sizeof($connections) == 0) {
+    // Break if max_time has been reached.
+    if (timer_read("millennium_mass_fetch") / 1000 > $max_time) {
       break;
     }
-
-    // Read from all open connections
-    foreach ($connections as $item_recnum => $connection) {
-      if (time() > $max_time) {
-        break;
-      }
-      if (!isset($connection) || $connection === false ) {
-        unset($connections[$item_recnum]);
-        continue;
-      }
-
-      // Try to read a chunk. $chunk will be empty if no data waiting, or eof.
-      $chunk = fread($connection, 1024);
-      if ($chunk) {
-        $buffer[$item_recnum] .= $chunk;
-        #drupal_set_message("millennium_fetch_recordpage_preload(): Read 1024 bytes for $item_recnum; size=".strlen($buffer[$item_recnum])." feof=".(feof($connection)?"yes":"no"));
-      }
-
-      // Handle finished connections
-      if (feof($connection)) {
-        list($headers, $data) = preg_split("/\r\n\r\n|\n\n/", $buffer[$item_recnum], 2);
-        #drupal_set_message("millennium_fetch_recordpage_preload(): feof reached for $item_recnum; data size=".strlen($data));
-        $key = "i{$item_recnum}". $mode;
-        $millennium_fetch_recordpage_cache[$key] = $data;
-        fclose($connection);
-        unset($connections[$item_recnum]);
-        unset($buffer[$item_recnum]);
-        #drupal_set_message("millennium_fetch_recordpage_preload(): Finished $item_recnum, key $key");
-        $open_connections--;
-        $tot_fetched++;
-      }
-    }
-  }
-  if (time() >= $max_time) {
-    watchdog("Millennium", "Prefetch aborting, reached time limit");
   }
-  watchdog("Millennium", "Prefetch successfully fetched @count records out of @limit in @time seconds",
-    array("@count" => $tot_fetched, "@limit" => $end_item_recnum - $start_item_recnum, "@time" => sprintf("%2.1f", microtime(true) - $start_time)));
-  #drupal_set_message(sprintf("millennium_fetch_recordpage_preload(): Fetched $tot_fetched records in %2.1f seconds", microtime(true) - $start_time));
-  return $tot_fetched;
+  #dpm($results);
+  return $results;
 }
 
 /**
- * Opens a nonblocking socket connection for simultaneous URL fetching, returns a file descriptor.
- * @param url the destination URL to open the socket to
- * @param post_data an optional string with post data to send in request's header
- * @param cookies string with cookies to send in request's header
- * @param referer string with referer to send in request's header
+ * Gets item information (bib number & MARC record) using the III's book cart.
+ * Recieves an array of item numbers and returns an array of found item data including bib number and MARC keyed by item number, and an unkeyed array of not found items.
+ * @param array $item_recnums An unkeyed array of item numbers = array('i100000', 'i100002', ...)
  */
-function _millennium_nonblocking_http_request($url, $post_data = false, $cookies = "", $referer = "") {
+function millennium_fetch_records_via_bookcart($item_recnums) {
+  static $headers = array();
 
-  //$follow_redirects = false;
+  // Start timer to measure average performance
+  timer_start("millennium_fetch_records_via_bookcart");
 
-  // Optional arguments... ($url,$post_data [$cookies[,$referer]])
-  if (func_num_args() >=3) {
-    $cookies = func_get_arg(2);
-  }
-  if (func_num_args() >=4 ) {
-    $referer = func_get_arg(3);
-  }
-  /*
-  if (func_num_args() >=5) {
-    $follow_redirects = func_get_arg(4);
-  }
+  $baseurl = millennium_get_real_baseurl();
+  // If called for first time, initiate a session and store the III_SESSION_ID cookie
+  if (sizeof($headers) == 0 ) {
+    // Get a session cookie
+    $result = drupal_http_request("{$baseurl}/record=i100000");
+    if (empty($result->headers["Set-Cookie"])) {
+      return array('found' => array(), 'not_found' => array());
+    }
+    $session_cookie = preg_replace(
+      '/.*(III_SESSION_ID=[a-zA-Z0-9\.]+).*/',
+      '\1',
+      $result->headers["Set-Cookie"]
+    );
+    $headers = array('Cookie' => $session_cookie);
+  }
+
+  // Add items from $item_recnums array to the cart
+  $path = '/search?/Xtest&searchscope=0&SORT=D/Xtest&searchscope=0&SORT=D&SUBKEY=test/1%2C7175%2C7175%2CE/2browse';
+  $post = "jumpref=Xtest&save=" . implode("&save=", $item_recnums) . "&save_func=save_marked";
+  $result = drupal_http_request("{$baseurl}{$path}", $headers, 'POST', $post);
+  #dpm($result);
+  #return;
+
+  // Get cart contents: only item and bib numbers
+  // From matches in list we can determine if some item numbers do not actually exist in the database.
+  $path = "/search/?/++export/1,-1,-1,B/export";
+  $result = drupal_http_request("{$baseurl}{$path}", $headers, 'GET');
+  $ok = preg_match_all(
+    '/name="save" value="(b[0-9]+)".*?browseEntryData.*?record=(i[0-9]+)/si',
+    $result->data,
+    $matches,
+    PREG_SET_ORDER
+  );
+  #dpm($matches);
+  #return;
+
+  // Start off assuming no items have been found
+  foreach ($item_recnums as $num) {
+    $not_found_items[$num] = $num;
+  }
+  // Store each found item's item <=> bib relationship
+  $found_items = array();
+  foreach ($matches as $match) {
+    $found_bib_recnum = $match[1];
+    $found_item_recnum = $match[2];
+    // Remove found items off $not_found_items list
+    unset($not_found_items[$found_item_recnum]);
+    $found_items[$found_item_recnum] = array(
+      'item_recnum' => $found_item_recnum,
+      'bib_recnum' => $found_bib_recnum
+    );
+  }
+  #dpm($item_to_bib);
+
+  // Get MARC for all!
+  $path = "/search/?.i100000/++export/1%2C-1%2C-1%2CB/export/";
+  $post = "email_addx=&email_subj=&ex_device=43&ex_format=50";
+  $result = drupal_http_request("{$baseurl}{$path}", $headers, 'POST', $post);
+  #dpm($result->data);
+  $ok = preg_match_all(
+    '/<pre>(.*?)<\/pre>/s',
+    $result->data,
+    $matches,
+    PREG_SET_ORDER
+  );
+  #dpm($matches);
+  // Assign marc to item numbers
+  $index = 0;
+  foreach ($found_items as $item => $dummy) {
+    $found_items[$item]['marc'] = $matches[$index][1];
+    $index++;
+  }
+  #dpm($found_items);
+
+  // Clear the cart
+  $path = "/search?/X/X/1,-1,-1,B/browse?clear_saves=1";
+  $dummy = drupal_http_request("{$baseurl}{$path}", $headers, 'GET');
+
+  // Read timer.
+  $elapsed = round(timer_read("millennium_fetch_records_via_bookcart") / 1000, 3);
+  #drupal_set_message("Tried to fetch:" . sizeof($item_recnums) . ". Found: " . sizeof($found_items) . ". Elapsed time: {$elapsed}s");
+
+  // Return results
+  $results = array('found' => $found_items, 'not_found' => $not_found_items);
+  #dpm($results);
+
+  return $results;
+
+  /* $results = array(
+      'found' => array(
+        'i100001' => array(
+          'item_recnum' => 'i100001',
+          'bib_recnum' => 'b426763',
+          'marc' => 'LEADER 00000cam 2200000 a 4500 001 tec042...
+        ),
+        'i100003' => array(...)
+        [...]
+      ),
+      'not_found' => array('i100000', 'i100005', [...])
+    );
   */
 
-  $connect_timeout = 5;
-
-  $url_parsed = parse_url($url);
-  $host = $url_parsed["host"];
-  $port = @$url_parsed["port"];
-  $userAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)";
-  if ($port == 0 || $port == "") {
-    $port = 80;
-  }
-  $path = $url_parsed["path"];
-  if (@$url_parsed["query"] != "") {
-    $path .= "?".$url_parsed["query"];
-  }
-
-  // Open the non-blocking socket connection
-  $fp = @fsockopen($host, $port, $errno, $errstr, $connect_timeout);
-  if ($fp == false) {
-    #drupal_set_message("_millennium_nonblocking_http_request():: Couldn't open connection to URL: $url  Errors: $errstr ($errno)");
-    return(false);
-  }
-
-  // Build the request header
-  $out = ($post_data != "") ? "POST" : "GET";
-  $out.= " $path HTTP/1.0\r\nHost: $host\r\n$userAgent\r\nAccept: */*\r\n";
-  if ($post_data != "") {
-    $out .= "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: ".strlen($post_data)."\r\n";
-  }
-  if ($cookies != "") {
-    $out .= "Cookie: ". $cookies ."\r\n";
-  }
-  if ($referer != "") {
-    $out .= "Referer: ". $referer ."\r\n";
-  }
-  $out .= "\r\n";
-  if ($post_data != "") {
-    $out .= $post_data ."\r\n\r\n";
-  }
-
-  // Send the request header
-  //drupal_set_message("_millennium_nonblocking_http_request(): Sending this header to ".$host.":".$port." ==> ". $out);
-  $success = fwrite($fp, $out);
-  if ($success == false) {
-    //drupal_set_message("_millennium_nonblocking_http_request():: Couldn't send request to URL ".$url."!");
-    return(false);
-  }
-  socket_set_blocking($fp, 0);
-  return($fp);
 }
Index: millennium.module
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/millennium.module,v
retrieving revision 1.13.2.33.2.2.2.27
diff -u -r1.13.2.33.2.2.2.27 millennium.module
--- millennium.module	13 Oct 2009 19:49:45 -0000	1.13.2.33.2.2.2.27
+++ millennium.module	13 Oct 2009 22:11:41 -0000
@@ -318,7 +318,7 @@
  */
 function millennium_import_update_item($item_recnum, $force_update = true, $marc_text = null, $bib_recnum = null) {
 
-  #drupal_set_message("millennium_import_update_item($item_recnum): start");
+  #drupal_set_message("millennium_import_update_item($item_recnum, $force_update, $marc_text, $bib_recnum): start");
 
   // Check if item exists on server
   $item_exists = true;