Index: millennium.import.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.import.inc,v
retrieving revision 1.1.2.5
diff -u -r1.1.2.5 millennium.import.inc
--- millennium.import.inc	21 Oct 2009 21:47:18 -0000	1.1.2.5
+++ millennium.import.inc	27 Oct 2009 22:51:41 -0000
@@ -2,24 +2,61 @@
 // $Id: millennium.import.inc,v 1.1.2.5 2009/10/21 21:47:18 janusman Exp $
 
 /**
- * Gets a sequential number of records obeying PHP's max_execution_time setting.
- * @param array $item_recnums Array of item numbers to fetch.
+ * Run tests on different WebOPACs available on the web. Requires devel.module to be enabled.
  */
-function millennium_mass_fetch($item_recnums) {
-  $max_records_for_bookcart = 50; // This seems to be the limit for WebOPACs, do not change =)
+function millennium_mass_fetch_test($num_records = 20) {
+  $test_subjects = array(
+    // catnyp.nypl.org DOES NOT WORK. Sorry, @anarchivist! =)
+    #array('http://catnyp.nypl.org', 'b', 18134000),
+    #array('http://catnyp.nypl.org', 'i', 17820000),
+
+    array('http://irma.aadl.org', 'i', 1500000),
+    array('http://elibrary.mel.org', 'i', 13500000),
+    array('http://millenium.itesm.mx', 'i', 100000),
+    array('http://www.consuls.org', 'b', 1580000),
+    array('http://irma.aadl.org', 'b', 1321000),
+    array('http://millenium.itesm.mx', 'b', 1179000),
+    array('http://elibrary.mel.org', 'b', 10340000),
+  );
+
+  foreach ($test_subjects as $subject) {
+    // Create array of records
+    $recnums = array();
+    $parsed_url = parse_url($subject[0]);
+    $base_url = "http://" . $parsed_url["host"];
+    for ($i = 0; $i < $num_records; $i++) {
+      $recnums[] = $subject[1] . ($subject[2] + $i);
+    }
 
-  if (ini_get('max_execution_time')<20) {
-    return false;
+    // Run test for a single URL
+    timer_start("millennium_mass_fetch_test");
+    $fetched = millennium_mass_fetch($recnums, $base_url);
+
+    // Report
+    $elapsed = sprintf("%2.2f", timer_read("millennium_mass_fetch_test") / 1000);
+    $recs_per_sec = sprintf("%2.2f", $num_records / (timer_read("millennium_mass_fetch_test")/1000));
+    $pct_found = sprintf("%2.1f", (sizeof($fetched["found"]) / $num_records) * 100);
+    drupal_set_message("{$base_url} : {$num_records} recs ({$pct_found}% found) in {$elapsed} seconds ({$recs_per_sec} rec/sec)");
+    dpm($fetched);
   }
+}
+
+/**
+ * Gets a sequential number of records obeying PHP's max_execution_time setting.
+ * @param array $recnums Array of item or bib numbers to fetch.
+ */
+function millennium_mass_fetch($recnums, $base_url = false) {
+  $max_records_for_bookcart = 50; // This seems to be the limit for WebOPACs, do not change =)
 
   // Don't devote more than 75% of max_execution_time to prefetch
   $max_time = ini_get('max_execution_time') * 0.75;
+  timer_start("millennium_mass_fetch");
 
   // Chunk array into groups of $max_records_for_bookcart
-  $chunks = array_chunk($item_recnums, $max_records_for_bookcart);
+  $chunks = array_chunk($recnums, $max_records_for_bookcart);
   $results = array('found' => array(), 'not_found' => array());
   foreach ($chunks as $chunk) {
-    $result = millennium_fetch_records_via_bookcart($chunk);
+    $result = millennium_fetch_records_via_bookcart($chunk, true, $base_url);
     $results['found'] = array_merge($results['found'], $result['found']);
     $results['not_found'] = array_merge($results['not_found'], $result['not_found']);
 
@@ -35,110 +72,164 @@
 /**
  * Gets item information (bib number & MARC record) using the III's book cart.
  * Recieves an array of item numbers and returns an array of found item data including bib number and MARC keyed by item number, and an unkeyed array of not found items.
- * @param array $item_recnums An unkeyed array of item numbers = array('i100000', 'i100002', ...)
+ * @param array $recnums An unkeyed array of item or bib numbers = array('i123456', 'b123456', ...)
  * @param bool $complete_holdings A flag that specifies if an extra request should be done if the holdings table is incomplete.
  */
-function millennium_fetch_records_via_bookcart($item_recnums, $complete_holdings = true) {
+function millennium_fetch_records_via_bookcart($recnums, $complete_holdings = true, $base_url = false) {
   static $headers = array();
+  static $debug = false;
 
   // Start timer to measure average performance
   timer_start("millennium_fetch_records_via_bookcart");
 
-  $baseurl = millennium_get_real_baseurl();
+  if ($base_url == false) {
+    $base_url = millennium_get_real_baseurl();
+    $base_url_parsed = parse_url($base_url);
+  }
+
   // If called for first time, initiate a session and store the III_SESSION_ID cookie
   if (sizeof($headers) == 0 ) {
     // Try to get from cache first
-    $cached = cache_get("millennium_fetch_records_via_bookcart_headers");
+    $cid = "millennium_fetch_records_via_bookcart_headers_" . $base_url_parsed['host'];
+    $cached = cache_get($cid);
     if ($cached->data) {
       $headers = $cached->data;
     } else {
+      $headers = array();
       // Get a session cookie
-      $result = drupal_http_request("{$baseurl}/search/X");
-      #dpm($result);
+      $result = drupal_http_request("{$base_url}/search*eng/X");
       if (empty($result->headers["Set-Cookie"])) {
         return array('found' => array(), 'not_found' => array());
       }
-      $session_cookie = preg_replace(
-        '/.*(III_SESSION_ID=[a-zA-Z0-9\.]+).*/',
-        '\1',
-        $result->headers["Set-Cookie"]
-      );
-      $headers = array('Cookie' => $session_cookie);
+      // Find cookies and put them into $headers
+      if (preg_match_all('/(SESSION_SCOPE=[0-9]+|III_SESSION_ID=[a-zA-Z0-9\.]+)/', $result->headers["Set-Cookie"], $matches)) {
+        $headers['Cookie'] = implode("; ", $matches[1]);
+      }
+
       // Cache this session cookie for 5 minutes
-      cache_set("millennium_fetch_records_via_bookcart_headers", $headers, 'cache', time() + 300);
+      cache_set($cid, $headers, 'cache', time() + 300);
     }
   }
 
   // Issue N requests to add items to the bookcart, 25 records at a time
-  $chunks = array_chunk($item_recnums, 25);
+  $chunks = array_chunk($recnums, 25);
   foreach ($chunks as $chunk) {
-    // Add items from $item_recnums array to the cart
-    $path = '/search?/X&searchscope=0&SORT=D/X&searchscope=0&SORT=D&SUBKEY=/1%2C7175%2C7175%2CE/2browse';
+    // Add items from $recnums array to the cart
+    $path = '/search*eng/X///';
     $post = "jumpref=X&save=" . implode("&save=", $chunk) . "&save_func=save_marked";
-    $result = drupal_http_request("{$baseurl}{$path}", $headers, 'POST', $post);
+    $result = drupal_http_request("{$base_url}{$path}", $headers, 'POST', $post);
+    if ($debug) {
+      drupal_set_message("Added items to bookcart response: path = {$base_url}{$path}");
+      dpm($result);
+    }
   }
-  #dpm($result);
-  #return;
 
   // Get cart contents: only item and bib numbers
   // From matches in list we can determine if some item numbers do not actually exist in the database.
-  $path = "/search/?/++export/1,-1,-1,B/export";
-  $result = drupal_http_request("{$baseurl}{$path}", $headers, 'GET');
-  $ok = preg_match_all(
-    '/name="save" value="(b[0-9]+)".*?browseEntryData.*?record=(i[0-9]+)/si',
-    $result->data,
-    $matches,
-    PREG_SET_ORDER
-  );
-  #dpm($matches);
-  #return;
+  $path = "/search*eng/?/++export/1,-1,-1,B/export";
+  $matches = array();
+  while (TRUE) {
+    // Repeat until no "next" pager link found
+    $result = drupal_http_request("{$base_url}{$path}", $headers, 'GET');
+    if ($debug) {
+      drupal_set_message("Cart contents and matches:");
+      dpm($result);
+    }
+    // Match the bib record, the given record and the title (for false positives, see $prev_title below).
+    $ok = preg_match_all(
+      '/name="save" value="(b[0-9]+)"[^>]+>.*?"browseEntryData">.*?record=([bi][0-9]+)[^>]+>([^<]*)<\/a>/si',
+      $result->data,
+      $tmp_matches,
+      PREG_SET_ORDER
+    );
+    if ($debug) {
+      drupal_set_message("One page of item<->bib matches from bookcart:");
+      dpm($tmp_matches);
+    }
+    if ($ok) {
+      $matches = array_merge($matches, $tmp_matches);
+    }
+    // Look for "next" pager: Looks like this
+    // <a href="/search/?/++export/13%2C-1%2C-1%2CB/export/">Next</a>
+    // <a href="/search~S1/?/++export/13%2C-1%2C-1%2CB/export/">Next</a>
+    $ok = preg_match('/<a href="(\/search[^"]+export\/[^"]+\/export\/)">Next/si', $result->data, $pager_link_matches);
+    if (!$ok) {
+      break;
+    }
+    $path = $pager_link_matches[1];
+  }
+  #if ($debug) dpm($matches);
 
   // Start off assuming no items have been found
-  foreach ($item_recnums as $num) {
+  foreach ($recnums as $num) {
     $not_found_items[$num] = $num;
   }
-  // Store each found item's item <=> bib relationship
+
   $found_items = array();
+
+  $prev_title = "";
   foreach ($matches as $match) {
     $found_bib_recnum = $match[1];
-    $found_item_recnum = $match[2];
+    $found_recnum = $match[2];
+    $found_title = $match[3];
+
+    /**
+     * The bookcart repeats non-existing bib records as repeated titles
+     * and (maybe) other empty fields. Ignore the item if the title repeats
+     * and the found_recnum is a bib record.
+     */
+    if ($prev_title == $found_title && substr($found_recnum, 0, 1) == "b") {
+      $prev_title = $found_title;
+      continue;
+    }
+    $prev_title = $found_title;
+
     // Remove found items off $not_found_items list
-    unset($not_found_items[$found_item_recnum]);
-    $found_items[$found_item_recnum] = array(
-      'item_recnum' => $found_item_recnum,
-      'bib_recnum' => $found_bib_recnum
+    unset($not_found_items[$found_recnum]);
+    $found_items[$found_recnum] = array(
+      'bib_recnum' => $found_bib_recnum,
+      'title' => $found_title,
     );
+    // If supplied recnum is an item record, store item <=> bib relationship
+    if (substr($found_recnum, 0, 1) == "i") {
+      $found_items[$found_recnum]['item_recnum'] = $found_recnum;
+    }
   }
-  #dpm($item_to_bib);
 
   // Get MARC for all!
-  $path = "/search/?/++export/1%2C-1%2C-1%2CB/export/";
+  $path = "/search*eng/?/++export/1,-1,-1/export/";
   $post = "email_addx=&email_subj=&ex_device=43&ex_format=50";
-  $result = drupal_http_request("{$baseurl}{$path}", $headers, 'POST', $post);
-  #dpm($result->data);
+  $result = drupal_http_request("{$base_url}{$path}", $headers, 'POST', $post);
   // Split records
   $ok = preg_match_all(
-    '/<div class="exportHeading">.*?<pre>(.*?)<\/pre>.*?(<table.*?bibItems.*?<\/table>)(?:.<center>.<form .*? action="(.*?)".*?<\/form>|)/si',
+      '/'
+    . '<div class="exportHeading">.*?<pre>(.*?)<\/pre>.*?'
+    . '(<table.*?bibItems.*?<\/table>|)'
+    . '(?:.<center>.<form .*? action="(.*?)".*?<\/form>|)'
+    . '/si',
     $result->data,
     $matches,
     PREG_SET_ORDER
   );
-  #dpm($matches);
+  if ($debug) {
+    drupal_set_message("MARC matches:");
+    dpm($matches);
+  }
   // Assign marc to item numbers
   $index = 0;
-  foreach ($found_items as $item => $dummy) {
-    $found_items[$item]['marc'] = $matches[$index][1];
+  foreach ($found_items as $recnum => $dummy) {
+    $found_items[$recnum]['marc'] = $matches[$index][1];
 
     // Add holdings information; determine if an extra request is needed
-    $this_bib_recnum = $found_items[$item]['bib_recnum'];
+    $this_bib_recnum = $found_items[$recnum]['bib_recnum'];
     if ($matches[$index][3] != "" && $complete_holdings == true) {
-      $found_items[$item]['holdings'] = millennium_get_holdings_info($this_bib_recnum);
+      $found_items[$recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum);
     } else {
-      $found_items[$item]['holdings'] = millennium_get_holdings_info($this_bib_recnum, $matches[$index][2]);
+      $found_items[$recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum, $matches[$index][2]);
     }
     $index++;
   }
-  
+
   // Grab XRECORD for item
   #foreach ($found_items as $item_recnum => $dummy) {
   #  $result = millennium_fetch_recordpage($item_recnum, 'xml');
@@ -150,27 +241,41 @@
 
   // Clear the cart
   $path = "/search?///1,-1,-1,B/browse?clear_saves=1";
-  $dummy = drupal_http_request("{$baseurl}{$path}", $headers, 'GET');
+  $dummy = drupal_http_request("{$base_url}{$path}", $headers, 'GET');
 
   // Read timer.
   $elapsed = round(timer_read("millennium_fetch_records_via_bookcart") / 1000, 3);
-  #drupal_set_message("Tried to fetch:" . sizeof($item_recnums) . ". Found: " . sizeof($found_items) . ". Elapsed time: {$elapsed}s");
+  #drupal_set_message("Tried to fetch:" . sizeof($recnums) . ". Found: " . sizeof($found_items) . ". Elapsed time: {$elapsed}s");
 
   // Return results
   $results = array('found' => $found_items, 'not_found' => $not_found_items);
-  #dpm($results);
+  if ($debug) {
+    drupal_set_message("Final results:");
+    dpm($results);
+  }
 
   return $results;
 
   /* $results = array(
       'found' => array(
-        'i100001' => array(
-          'item_recnum' => 'i100001',
+        'b100001' => array(
+          'bib_recnum' => 'b100001',
+          'marc' => 'LEADER 00000cam 2200000 a 4500 001 tec042...
+          'holdings' => '...'
+        ),
+        'i222222' => array(
+          'item_recnum' => 'i222222',
           'bib_recnum' => 'b426763',
           'marc' => 'LEADER 00000cam 2200000 a 4500 001 tec042...
+          'holdings' => array(
+            0 => array(
+              'location' => 'MTY 4to. Piso',
+              'classnumber' => 'QH541.15.B56 E95 2009',
+              'classvolume' => '',
+              'status' => 'CATALOGACION',
+            ),
+          ),
         ),
-        'i100003' => array(...)
-        [...]
       ),
       'not_found' => array('i100000', 'i100005', [...])
     );