Index: feedapi.module
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/feedapi/feedapi.module,v
retrieving revision 1.23.2.119.2.71
diff -u -p -r1.23.2.119.2.71 feedapi.module
--- feedapi.module	4 Aug 2009 15:00:08 -0000	1.23.2.119.2.71
+++ feedapi.module	10 Aug 2009 15:11:22 -0000
@@ -688,40 +688,20 @@ function feedapi_cron() {
 
   db_query('DELETE FROM {feedapi_stat} WHERE timestamp < %d', variable_get('cron_semaphore', FALSE) - FEEDAPI_CRON_STAT_LIFETIME);
 
-  // Initialize counters
-  $count = array(
-    '%feeds' => 0,
-    '%expired' => 0,
-    '%new' => 0,
-    '%updated' => 0,
-  );
+  $result = db_query_range("SELECT f.nid, n.uid FROM {feedapi} f JOIN {node} n ON n.vid = f.vid WHERE next_refresh_time < %d AND next_refresh_time <> %d ORDER BY next_refresh_time ASC", time(), FEEDAPI_CRON_NEVER_REFRESH, 0, FEEDAPI_CRON_FEEDS);
 
-  // We get feeds in small lots, this will save memory and have the process adjusting to the
-  // time limit even when we have many thousands of them.
-  $now = time();
-  $process = 0;
-  // The counter process will be > 0 if we've selected less feeds
-  while (!$process && feedapi_cron_time()) {
-    $process = FEEDAPI_CRON_FEEDS;
-    $result = db_query_range("SELECT f.nid, n.uid FROM {feedapi} f JOIN {node} n ON n.vid = f.vid WHERE next_refresh_time < %d AND next_refresh_time <> %d ORDER BY next_refresh_time ASC", $now, FEEDAPI_CRON_NEVER_REFRESH, 0, FEEDAPI_CRON_FEEDS);
-
-    while (feedapi_cron_time() && $feed = db_fetch_object($result)) {
-      $user = user_load(array('uid' => $feed->uid));
-      // Call the refresh process for each feed and store counters
-      $counter = feedapi_invoke('refresh', $feed, TRUE);
-      if ($counter) {
-        foreach ($counter as $name => $value) {
-          $count['%'. $name] += $value;
-        }
-      }
-      $count['%feeds']++;
-      $process--;
-    }
+  while ($feed = db_fetch_object($result)) {
+    // Call the refresh process for each feed and store counters
+    feedapi_invoke('refresh', $feed, TRUE);
   }
 
   // Loads back the logged in user
   $user = $original_user;
   session_save_session(TRUE);
+
+  // Process all batches set by the refresh process.
+  // We must register the process as a shutdown function. Otherwise we will mess up the cron flow. 
+  register_shutdown_function('batch_process', 'admin/reports/status');
 }
 
 /**
@@ -1166,11 +1146,6 @@ function _feedapi_invoke($op, &$feed, $p
  * @ TODO Fix: This may loop forever when a feed has no processors
  */
 function _feedapi_invoke_refresh(&$feed, $param) {
-  $timestamp = variable_get('cron_semaphore', FALSE) !== FALSE ? variable_get('cron_semaphore', FALSE) : time();
-
-  $counter = array();
-  timer_start('feedapi_'. $feed->nid);
-  $memory_usage = memory_get_usage();
   $cron = $param;
 
   // Step 0: Check processors and grab settings
@@ -1184,7 +1159,7 @@ function _feedapi_invoke_refresh(&$feed,
   $settings = feedapi_get_settings(NULL, $feed->vid);
 
   // Step 1: Force processors to delete old items and determine the max. create elements.
-  $counter['expired'] = feedapi_expire($feed, $settings);
+  feedapi_expire($feed, $settings);
 
   // Step 2: Get feed.
   $nid = $feed->nid;
@@ -1206,86 +1181,54 @@ function _feedapi_invoke_refresh(&$feed,
         drupal_set_message(t('Could not refresh feed.'), 'error');
       }
     }
-    return $counter;
+    return;
   }
 
-  // Step 4: Walk through the items and check duplicates, then save or update
+  // Step 4: Build the batch.
   $items = $feed->items;
-  $updated = 0;
-  $new = 0;
-  $half_done = FALSE;
+  $batch = array(
+    'title' => t('Processing feeds'),
+    'operations' => array()
+  );
 
-  // We check for time-out after each item
   foreach ($items as $index => $item) {
     // Call each item parser.
-    $item->is_updated = FALSE;
-    $item->is_new = FALSE;
     foreach ($feed->processors as $processor) {
-      $unique = module_invoke($processor, 'feedapi_item', 'unique', $item, $feed->nid, $settings['processors'][$processor]);
-      if ($unique === FALSE || is_numeric($unique)) {
-        if ($settings['update_existing'] == TRUE) {
-          module_invoke($processor, 'feedapi_item', 'update', $item, $feed->nid, $settings['processors'][$processor], $unique);
-          $item->is_updated = TRUE;
-        }
-      }
-      else {
-        // We have checked before for expired items, so just save it.
-        // if the item is already expired then do nothing
-        $items_delete = $settings['items_delete'];
-        $diff = abs(time() - (isset($item->options->timestamp) ? $item->options->timestamp : time()));
-        if ($diff > $items_delete && ($items_delete > FEEDAPI_NEVER_DELETE_OLD)) {
-          break;
-        }
-        $result = module_invoke($processor, 'feedapi_item', 'save', $item, $feed->nid, $settings['processors'][$processor]);
-        if ($result !== FALSE) {
-          $item->is_new = TRUE;
-        }
-      }
+      $batch['operations'][] = array('_feedapi_invoke_refresh_operation', array($processor, $item, $feed->nid, $settings));
     }
-    $new = $item->is_new ? $new + 1 : $new;
-    $updated = ($item->is_updated && !$item->is_new) ? $updated + 1 : $updated;
-
-    // Decision on time. If the exec time is greather than the user-set percentage of php max execution time
-    if ($cron && !feedapi_cron_time()) {
-      $half_done = ($new + $updated) == count($items) ? FALSE : TRUE;
-      break;
-    }
-    // Save the item status for further processing
-    $feed->items[$index] = $item;
   }
 
-  // Closing step: Call after refresh and update feed statistics
-  foreach (module_implements('feedapi_after_refresh') as $module) {
-    $func = $module .'_feedapi_after_refresh';
-    $func($feed);
-  }
+  // Step 5: Set the batch for later processing.
+  batch_set($batch);
 
   // Set next_refresh_time to FEEDAPI_CRON_NEVER_REFRESH if refresh_time is FEEDAPI_CRON_NEVER_REFRESH.
   $next_refresh_time = $settings['refresh_time'] == FEEDAPI_CRON_NEVER_REFRESH ? $settings['refresh_time'] : (time() + $settings['refresh_time']);
-  db_query("UPDATE {feedapi} SET next_refresh_time = %d, half_done = %d, hash = '%s' WHERE nid = %d", $next_refresh_time, $half_done, $feed->hash, $feed->nid);
-
-  // Log statistics.
-  _feedapi_store_stat($nid, 'update_times', time(), $timestamp);
-  _feedapi_store_stat($nid, 'new', $new, $timestamp);
-  _feedapi_store_stat($nid, 'download_num', count($items), $timestamp);
-  _feedapi_store_stat($nid, 'process_time', timer_read('feedapi_'. $feed->nid), $timestamp);
-  _feedapi_store_stat($nid, 'memory_increase', memory_get_usage() - $memory_usage, $timestamp);
-  _feedapi_store_stat($nid, 'next_refresh_time', $next_refresh_time, $timestamp);
+  db_query("UPDATE {feedapi} SET next_refresh_time = %d, hash = '%s' WHERE nid = %d", $next_refresh_time, $feed->hash, $feed->nid);
 
   if (!$cron) {
-    if ($new == 0 && $updated == 0) {
-      drupal_set_message(t('There are no new items in the feed.'), 'status');
-    }
-    else {
-      drupal_set_message(t("%new new item(s) were saved. %updated existing item(s) were updated.", array("%new" => $new, "%updated" => $updated)));
+    // Only process the batch here if this request isn't made during cron.
+    // During cron the batch processing gets done in hook_cron().
+    batch_process('admin/content/feed');
+    drupal_set_message(t('The feed was refreshed.'), 'status');
+  }
+}
+
+function _feedapi_invoke_refresh_operation($processor, $item, $nid, $settings) {
+  $unique = module_invoke($processor, 'feedapi_item', 'unique', $item, $nid, $settings['processors'][$processor]);
+  if ($unique === FALSE || is_numeric($unique)) {
+    if ($settings['update_existing'] == TRUE) {
+      module_invoke($processor, 'feedapi_item', 'update', $item, $nid, $settings['processors'][$processor], $unique);
     }
-    // @ TODO what value to return here?
   }
   else {
-    // Update and return counter
-    $counter['new'] = $new;
-    $counter['updated'] = $updated;
-    return $counter;
+    // We have checked before for expired items, so just save it.
+    // If the item is already expired then do nothing.
+    $items_delete = $settings['items_delete'];
+    $diff = abs(time() - (isset($item->options->timestamp) ? $item->options->timestamp : time()));
+    if ($diff > $items_delete && ($items_delete > FEEDAPI_NEVER_DELETE_OLD)) {
+      break;
+    }
+    module_invoke($processor, 'feedapi_item', 'save', $item, $nid, $settings['processors'][$processor]);
   }
 }