Index: millennium.import.inc =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/millennium/Attic/millennium.import.inc,v retrieving revision 1.1.2.6 diff -u -r1.1.2.6 millennium.import.inc --- millennium.import.inc 30 Oct 2009 21:47:16 -0000 1.1.2.6 +++ millennium.import.inc 2 Dec 2009 20:13:08 -0000 @@ -25,11 +25,11 @@ array('http://library.dcccd.edu', 'b', 1100000), array('http://irma.aadl.org', 'b', 1321000), array('http://catnyp.nypl.org', 'b', 18134000), - #array('http://www.consuls.org', 'b', 1580000), - #array('http://elibrary.mel.org', 'b', 10340000), - #array('http://catalogue.gvpl.ca' , 'b', 1100000), + array('http://www.consuls.org', 'b', 1580000), #array('http://eleanor.lib.gla.ac.uk', 'b', 2661000), - #array('http://library.hku.hk', 'b', 1100000), + #array('http://elibrary.mel.org', 'b', 10340000), // SLOW! + #array('http://catalogue.gvpl.ca' , 'b', 1100000), // SLOW! + #array('http://library.hku.hk', 'b', 1100000), // SLOW ); foreach ($test_subjects as $subject) { @@ -119,6 +119,8 @@ if (!millennium_ping($base_url)) { return array('found' => array(), 'not_found' => $recnums); } + $found_items = array(); + $not_found_items = array(); // If called for first time, initiate a session and store the III_SESSION_ID cookie if (!isset($headers_static_cache[$base_url_parsed['host']])) { @@ -145,6 +147,10 @@ } else { $headers = $headers_static_cache[$base_url_parsed['host']]; } + + // Clear the cart + $path = "/search?///1,-1,-1,B/browse?clear_saves=1"; + $dummy = drupal_http_request("{$base_url}{$path}", $headers, 'GET'); // Issue N requests to add items to the bookcart, 25 records at a time (apparent maximum) $chunks = array_chunk($recnums, 25); @@ -166,7 +172,7 @@ // Get cart contents: only item and bib numbers // From matches in list we can determine if some item numbers do not actually exist in the database. $path = "/search*eng/X/++export/1,-1,-1,B/export"; - $matches = array(); + $bookcart_matches = array(); while (TRUE) { // Repeat until no "next" pager link found $result = drupal_http_request("{$base_url}{$path}", $headers, 'GET'); @@ -175,6 +181,8 @@ dpm($result); } // Match the bib record, the given record and the title (for false positives, see $prev_title below). + // Sample: + //   The city trilogy : five jade disks, defenders of the dragon Chang Hsi-kuo ; translated from the Chinese by John Balcom. : Chang, S. K.; CCM GENERAL:DISPONIBLE, CEM GENERAL:DISPONIBLE, CSF GENERAL:DISPONIBLE, SIN GENERAL:DISPONIBLE   $ok = preg_match_all( '/name="save" value="(b[0-9]+)"[^>]+>.*?"browseEntryData">.*?record=([bi][0-9]+)[^>]+>([^<]*)<\/a>/si', $result->data, @@ -186,7 +194,7 @@ dpm($tmp_matches); } if ($ok) { - $matches = array_merge($matches, $tmp_matches); + $bookcart_matches = array_merge($bookcart_matches, $tmp_matches); } // Look for "next" pager: Found code like this: // Next @@ -198,37 +206,28 @@ } $path = $pager_link_matches[1]; } - #if ($debug) dpm($matches); - + if ($debug) { + drupal_set_message("Finished paging thru bookcart. These are all the item<->bib matches from bookcart:"); + dpm($bookcart_matches); + } + // Start off assuming no items have been found foreach ($recnums as $num) { $not_found_items[$num] = $num; } - - $found_items = array(); - - $prev_title = ""; - foreach ($matches as $match) { - $found_bib_recnum = $match[1]; + + // Order match data into an array + // NOTE: At this point some matches are false, since the bookcart export shows + // items for record numbers that don't actually exist. These will be dealt + // with below, by checking MARC against the title shown and stored here. + foreach ($bookcart_matches as $match) { $found_recnum = $match[2]; - $found_title = $match[3]; - - /** - * The bookcart repeats non-existing bib records as repeated titles - * and (maybe) other empty fields. Ignore the item if the title repeats - * and the found_recnum is a bib record. - */ - if ($prev_title == $found_title && substr($found_recnum, 0, 1) == "b") { - $prev_title = $found_title; - continue; - } - $prev_title = $found_title; - // Remove found items off $not_found_items list unset($not_found_items[$found_recnum]); + // Store title and recnum $found_items[$found_recnum] = array( - 'bib_recnum' => $found_bib_recnum, - 'title' => $found_title, + 'bib_recnum' => $match[1], + 'title' => trim($match[3]), ); // If supplied recnum is an item record, store item <=> bib relationship if (substr($found_recnum, 0, 1) == "i") { @@ -248,26 +247,61 @@ . '(?:.
.
|)' . '/si', $result->data, - $matches, + $marc_matches, PREG_SET_ORDER ); if ($debug) { + drupal_set_message("MARC request result:"); + dpm($result); drupal_set_message("MARC matches:"); - dpm($matches); + dpm($marc_matches); } - // Assign marc to item numbers - $index = 0; - foreach ($found_items as $recnum => $dummy) { - $found_items[$recnum]['marc'] = $matches[$index][1]; - + + // Assign marc to the record data in $found_items + // As mentioned above, $found_items contains some false positives, so check + // the title stored above against the marc data before storing. False + // positives get sent to the not_found_items array. + $found_items_indexes = array_keys($found_items); + $found_items_index = 0; // start at first item in $found_items + foreach ($marc_matches as $match) { + $marc_text = $match[1]; + $current_found_item = $found_items[$found_items_indexes[$found_items_index]]; + $current_recnum = $found_items_indexes[$found_items_index]; + // Check if current found_item title coincides with title in this MARC + while (1) { + // TODO: The next line is too simplistic; maybe it needs to parse the marc + // and only use the 240|a and 245|a portion to compare to the title in + // $current_found_item["title"] + if (stripos(millennium_only_letters($marc_text), drupal_substr(millennium_only_letters($current_found_item["title"]), 0, 8)) === false) { + // No; then remove this found_item and skip to the next one, check again + if ($debug) { + drupal_set_message("$current_recnum : Found item and marc not in sync! Current found_item:"); + dpm($current_found_item); + drupal_set_message("Found_item title: " . millennium_only_letters($current_found_item["title"])); + drupal_set_message("MARC: " . millennium_only_letters($marc_text)); + } + $not_found_items[$current_recnum] = $current_recnum; + unset($found_items[$current_recnum]); + $found_items_index++; + $current_found_item = $found_items[$found_items_indexes[$found_items_index]]; + $current_recnum = $found_items_indexes[$found_items_index]; + + } else { + if ($debug) { + drupal_set_message("$current_recnum : Found item and marc in sync!"); + } + $found_items[$current_recnum]['marc'] = $marc_text; + break; + } + } // Add holdings information; determine if an extra request is needed - $this_bib_recnum = $found_items[$recnum]['bib_recnum']; - if ($matches[$index][3] != "" && $complete_holdings == true) { - $found_items[$recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum, null, $base_url); + $this_bib_recnum = $found_items[$current_recnum]['bib_recnum']; + if ($match[3] != "" && $complete_holdings == true) { + $found_items[$current_recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum, null, $base_url); } else { - $found_items[$recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum, $matches[$index][2], $base_url); + $found_items[$current_recnum]['holdings'] = millennium_get_holdings_info($this_bib_recnum, $match[2], $base_url); } - $index++; + $found_items_index++; } // Grab XRECORD for item @@ -322,3 +356,9 @@ */ } + +function millennium_only_letters($string) { + $new = preg_replace("/\|[a-z]/", "", $string); + $new = drupal_strtolower(preg_replace("/[^A-Za-z]/", "", $new)); + return $new; +}