Index: review/simpletest/pifr_simpletest.module
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/review/simpletest/pifr_simpletest.module,v
retrieving revision 1.3
diff -u -r1.3 pifr_simpletest.module
--- review/simpletest/pifr_simpletest.module	4 Dec 2009 20:29:08 -0000	1.3
+++ review/simpletest/pifr_simpletest.module	12 Dec 2009 09:42:18 -0000
@@ -19,8 +19,9 @@
 define('PIFR_SERVER_TEST_RESULT_SYNTAX', 6);
 define('PIFR_SERVER_TEST_RESULT_INSTALL', 7);
 define('PIFR_SERVER_TEST_RESULT_TEST', 8);
-define('PIFR_SERVER_TEST_RESULT_FAIL', 9);
-define('PIFR_SERVER_TEST_RESULT_PASS', 10);
+define('PIFR_SERVER_TEST_RESULT_PROCESS', 9);
+define('PIFR_SERVER_TEST_RESULT_FAIL', 10);
+define('PIFR_SERVER_TEST_RESULT_PASS', 11);
 
 /**
  * Implementation of hook_pifr_info().
Index: review/simpletest/pifr_simpletest.server.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/review/simpletest/pifr_simpletest.server.inc,v
retrieving revision 1.11
diff -u -r1.11 pifr_simpletest.server.inc
--- review/simpletest/pifr_simpletest.server.inc	11 Dec 2009 02:54:19 -0000	1.11
+++ review/simpletest/pifr_simpletest.server.inc	12 Dec 2009 09:42:18 -0000
@@ -33,6 +33,12 @@
       'summary' => 'Failed to run tests on @environment: @reason.',
       'confirmation' => 'pifr_simpletest',
     ));
+    $this->steps['process'] = array_merge($this->steps['process'], array(
+      'title' => 'crash detected',
+      'active title' => 'detect a crashed test',
+      'description' => 'SimpleTest may have crashed due to a fatal error in the tests. Review your code for fatal errors.',
+      'confirmation' => 'pifr_simpletest',
+    ));
     $this->steps['fail'] = array_merge($this->steps['fail'], array(
       'title' => 'detect a failing test',
       'active title' => 'detect a failing test',
Index: review/simpletest/pifr_simpletest.client.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/review/simpletest/pifr_simpletest.client.inc,v
retrieving revision 1.15
diff -u -r1.15 pifr_simpletest.client.inc
--- review/simpletest/pifr_simpletest.client.inc	10 Dec 2009 23:09:51 -0000	1.15
+++ review/simpletest/pifr_simpletest.client.inc	12 Dec 2009 09:42:18 -0000
@@ -16,6 +16,13 @@
 class pifr_client_review_pifr_simpletest extends pifr_client_review_pifr_drupal {
 
   /**
+   * Keep a results array.
+   * 
+   * @var array
+   */
+  protected $result = array();
+
+  /**
    * Run SimpleTest tests.
    */
   protected function review() {
@@ -68,11 +75,17 @@
     // Run tests from command line.
     chdir($this->checkout_directory);
 
-    $url = escapeshellarg(url('', array('absolute' => TRUE)) . 'checkout');
-    if (!$this->exec(PIFR_CLIENT_PHP . ' ./scripts/run-tests.sh --concurrency ' . PIFR_CLIENT_CONCURRENCY .
-      ' --php ' . PIFR_CLIENT_PHP . ' --url ' . $url . ' ' . $this->test_list())) {
-      $this->set_error(array('@reason' => t('failed during invocation of run-tests.sh')));
-      return;
+    // Enumerate test classes and run the tests.
+    if ($this->enumerate_tests()) {
+      $command = PIFR_CLIENT_PHP . ' ./scripts/run-tests.sh';
+      $command .= ' --concurrency ' . escapeshellarg(PIFR_CLIENT_CONCURRENCY);
+      $command .= ' --php ' . escapeshellarg(PIFR_CLIENT_PHP);
+      $command .= ' --url ' . escapeshellarg(url('', array('absolute' => TRUE)) . 'checkout');
+      $command .= ' ' . $this->get_tests_argument();
+
+      if (!$this->exec($command)) {
+        $this->set_error(array('@reason' => t('failed during invocation of run-tests.sh')));
+      }
     }
 
     chdir($this->original['directory']);
@@ -80,6 +93,8 @@
 
   /**
    * Perform Drupal 6 SimpleTest installation.
+   * 
+   * @return boolean Success
    */
   protected function test_prepare() {
     // Apply core patch.
@@ -102,64 +117,111 @@
   }
 
   /**
+   * Enumerate selected tests so none are unaccounted for in the
+   * results. Create placeholders in the results array.
+   * 
+   * @return boolean Success
+   */
+  protected function enumerate_tests() {
+    // Use a fixed test list if running in debug mode.
+    if (PIFR_DEBUG) {
+      $this->test['review']['argument']['tests'] = array('NonDefaultBlockAdmin');
+    }
+    if (!empty($this->test['review']['argument']['tests'])) {
+      // Initialize a results sub-array for every test class requested.
+      foreach ($this->test['review']['argument']['tests'] as $class) {
+        $this->result['data'][$class] = array(
+          'test_name' => $class,
+          'pass' => 0,
+          'fail' => 0,
+          'exception' => 0,
+          'assertions' => array(),
+        );
+      }
+      return TRUE;
+    }
+    $command = PIFR_CLIENT_PHP . ' ./scripts/run-tests.sh --list';
+    $command .= ' --php ' . escapeshellarg(PIFR_CLIENT_PHP);
+    $command .= ' --url ' . escapeshellarg(url('', array('absolute' => TRUE)) . 'checkout');
+    $command .= ' ' . $this->get_tests_argument();
+    $output = $this->exec($command, FALSE, TRUE);
+
+    if (is_array($output)) {
+      // Initialize a results sub-array for every test class found.
+      foreach ($output as $line) {
+        if (preg_match("/\((\w+)\)$/", $line, $matches)) {
+          $this->result['data'][$matches[1]] = array(
+            'test_name' => $matches[1],
+            'pass' => 0,
+            'fail' => 0,
+            'exception' => 0,
+            'assertions' => array(),
+          );
+        }
+      }
+    }
+
+    if ($output === FALSE || empty($this->result)) {
+      $this->set_error(array('@reason' => 'failed to enumerate tests'));
+      return FALSE;
+    }
+    return TRUE;
+  }
+
+  /**
    * Create the test argument for run-tests.sh.
    *
    * @return string Tests argument to be appended to run-test.sh command.
    */
-  protected function test_list() {
+  protected function get_tests_argument() {
     if (!empty($this->test['review']['argument']['tests'])) {
-      return '--class ' . implode(',', $this->test['review']['argument']['tests']);
+      return '--class ' . escapeshellarg(implode(',', $this->test['review']['argument']['tests']));
     }
     elseif (!empty($this->test['review']['argument']['modules'])) {
-      $args = array();
+      $paths = array();
       foreach ($this->test['review']['argument']['modules'] as $module) {
-        $args[] = $this->module_path($module) . '/' . $module . '.test';
+        $paths[] = $this->module_path($module) . '/' . $module . '.test';
       }
-      return '--file ' . implode(',', $args);
+      return '--file ' . escapeshellarg(implode(',', $paths));
     }
-    elseif (PIFR_DEBUG) {
-      return '--class NonDefaultBlockAdmin';
+    else {
+      return '--all';
     }
-    return '--all';
   }
 
   /**
-   * Add the SimpleTest specific results.
+   * Process the test results.
    */
-  public function get_result() {
-    $result = parent::get_result();
+  public function process() {
+    // Add in general results.
+    $this->result += parent::get_result();
 
     // If result code is one greater then number of operations then all the
     // operations passed, so the test results need to be added.
     $fail_code = count($this->operations) + 1;
-    if ($result['code'] == $fail_code) {
-      $result['details'] = array(
+    if ($this->result['code'] == $fail_code) {
+      $this->result['details'] = array(
         '@pass' => 0,
         '@fail' => 0,
         '@exception' => 0,
       );
-      $result['data'] = array();
+
+      // Make a checklist out of the enumerated test classes,
+      // and verify that they are found in the database records.
+      $checklist = $this->result['data'];
 
       // Cycle through assertions to fill in results.
       $assertions = $this->database->query('SELECT * FROM {simpletest}');
       $non_pass = 0;
       foreach ($assertions as $assertion) {
         // Fill in details with summary of assertion counts.
-        $result['details']['@' . $assertion['status']]++;
-
-        // Initialize each test class.
-        if (!isset($result['data'][$assertion['test_class']])) {
-          $result['data'][$assertion['test_class']] = array(
-            'test_name' => $assertion['test_class'],
-            'pass' => 0,
-            'fail' => 0,
-            'exception' => 0,
-            'assertions' => array(),
-          );
-        }
+        $this->result['details']['@' . $assertion['status']]++;
 
         // Add up summary totals for test class.
-        $result['data'][$assertion['test_class']][$assertion['status']]++;
+        $this->result['data'][$assertion['test_class']][$assertion['status']]++;
+
+        // Remove the test class from the checklist.
+        unset($checklist[$assertion['test_class']]);
 
         // Store non-pass assertion until reaching maximum number.
         if ($assertion['status'] != 'pass' && $non_pass < PIFR_CLIENT_MAX_ASSERTIONS) {
@@ -171,19 +233,37 @@
           $assertion['file'] = basename($assertion['file']);
 
           // Add assertion to set of non-pass assertions for the class.
-          $result['data'][$assertion['test_class']]['assertions'][] = $assertion;
+          $this->result['data'][$assertion['test_class']]['assertions'][] = $assertion;
 
           // Keep a record of the number of non-pass assertions.
           $non_pass++;
         }
       }
 
+      // Check for incomplete results.
+      foreach ($this->result['data'] as $data) {
+        if (isset($checklist[$data['test_name']])) {
+          // Add a failure if the test class did not return a result.
+          $this->result['data'][$data['test_name']]['fail']++;
+          $this->result['details']['@fail']++;
+          $this->set_error(array('@reason' => 'detected a truncated test result'));
+        }
+      }
+
       // Remove class name keys.
-      $result['data'] = array_values($result['data']);
+      $this->result['data'] = array_values($this->result['data']);
 
-      // Assign the code bassed on the test results.
-      $result['code'] = ($result['details']['@fail'] + $result['details']['@exception'] ? $fail_code : $fail_code + 1);
+      // Assign the code based on the test results.
+      $this->result['code'] = ($this->result['details']['@fail'] + $this->result['details']['@exception'] ? $fail_code : $fail_code + 1);
     }
-    return $result;
+  }
+
+  /**
+   * Return the results.
+   * 
+   * @return array Results
+   */
+  public function get_result() {
+    return $this->result;
   }
 }
Index: review/client.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/review/client.inc,v
retrieving revision 1.16
diff -u -r1.16 client.inc
--- review/client.inc	11 Dec 2009 22:51:43 -0000	1.16
+++ review/client.inc	12 Dec 2009 09:42:18 -0000
@@ -601,9 +601,10 @@
    * @param string $command Command to execute.
    * @param boolean $capture_stderr Capture the stderr by mapping it to the
    *   stdout and collecting it on failure in the review log.
+   * @param boolean $return_output 
    * @return boolean TRUE if success, otherwise FALSE.
    */
-  public static function exec($command, $capture_stderr = TRUE) {
+  public static function exec($command, $capture_stderr = TRUE, $return_output = FALSE) {
     if ($capture_stderr) {
       $command .= ' 2>&1';
     }
@@ -614,6 +615,6 @@
       self::$instance->log('Command [' . $command . '] failed with ' . ($output ? 'output: ' . $output : 'no output') . '.');
       return FALSE;
     }
-    return TRUE;
+    return $return_output ? $output : TRUE;
   }
 }
Index: review/server.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/review/server.inc,v
retrieving revision 1.12
diff -u -r1.12 server.inc
--- review/server.inc	11 Dec 2009 03:28:20 -0000	1.12
+++ review/server.inc	12 Dec 2009 09:42:18 -0000
@@ -90,7 +90,7 @@
     'apply' => array(
       'title' => 'non-applicable patch',
       'active title' => 'detect a non-applicable patch',
-      'description' => 'Ensure the patch applies to the lastest checkotu of the code-base.',
+      'description' => 'Ensure the patch applies to the lastest checkout of the code-base.',
       'help' => 'The client failed to @type which is induced using the !patch.',
       'summary' => 'Unable to apply patch @filename.',
       'confirmation' => 'pifr_server',
@@ -119,6 +119,14 @@
       'summary' => 'Failed to review on @environment: @reason.',
       'confirmation' => FALSE,
     ),
+    'process' => array(
+      'title' => 'incomplete results',
+      'active title' => 'detect incomplete results',
+      'description' => 'The test results are incomplete and may not be reliable.',
+      'help' => 'The client failed to @type which is induced using the !patch.',
+      'summary' => 'Found incomplete test results on @environment.',
+      'confirmation' => FALSE,
+    ),
     'fail' => array(
       'title' => 'review failed',
       'active title' => 'detect a failing review',
Index: client/pifr_client.review.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/project_issue_file_review/client/pifr_client.review.inc,v
retrieving revision 1.11
diff -u -r1.11 pifr_client.review.inc
--- client/pifr_client.review.inc	8 Dec 2009 19:01:29 -0000	1.11
+++ client/pifr_client.review.inc	12 Dec 2009 09:42:18 -0000
@@ -112,6 +112,7 @@
       'default' => t('Default - Drupal 7, full suite'),
       'quick' => t('Quick - Drupal 7, NonDefaultBlockAdmin test case'),
       'fail' => t('Fail - Drupal 7, full suite, PHP exceptions'),
+      'fail_fatal' => t('Fatal - Drupal 7, full suite, PHP Fatal Error in a test'),
       'd6' => t('Drupal 6 - Several dependencies, full suite'),
     ),
   );
@@ -140,16 +141,20 @@
 
     variable_del('pifr_client_test');
   }
+  else {
+    form_set_error('type', t('Could not load pre-built test.'));
+  }
 }
 
 /**
  * Get a pre-built test, given the $type.
  *
- * @param string $type Test type, 'default', 'quick', 'fail', 'd6'.
+ * @param string $type Test type, 'default', 'quick', 'fail', 'fail_fatal',
+ *   or 'd6'.
  * @return array Pre-built test information array.
  */
 function pifr_client_review_pre_built_test($type) {
-  if ($type == 'default' || $type == 'quick' || $type == 'fail') {
+  if ($type == 'default' || $type == 'quick' || $type == 'fail' || $type == 'fail_fatal') {
     // Provide Drupal HEAD repository and blank patch.
     $test = array(
       'test_id' => '17',
@@ -186,6 +191,11 @@
         url(drupal_get_path('module', 'pifr_simpletest') . '/confirmation/fail.patch', array('absolute' => TRUE)),
       );
     }
+    elseif ($type == 'fail_fatal') {
+      $test['files'] = array(
+        url(drupal_get_path('module', 'pifr_simpletest') . '/confirmation/fail_fatal.patch', array('absolute' => TRUE)),
+      );
+    }
   }
   elseif ($type == 'd6') {
     // Contrib test with dependencies.
Index: review/simpletest/confirmation/fail_fatal.patch
===================================================================
RCS file: review/simpletest/confirmation/fail_fatal.patch
diff -N review/simpletest/confirmation/fail_fatal.patch
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ review/simpletest/confirmation/fail_fatal.patch	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,24 @@
+Index: modules/simpletest/simpletest.test
+===================================================================
+RCS file: /cvs/drupal/drupal/modules/simpletest/simpletest.test,v
+retrieving revision 1.36
+diff -u -r1.36 simpletest.test
+--- modules/simpletest/simpletest.test	16 Oct 2009 03:01:54 -0000	1.36
++++ modules/simpletest/simpletest.test	10 Dec 2009 22:59:57 -0000
+@@ -36,6 +36,16 @@
+       parent::setUp();
+     }
+   }
++  
++  /**
++   * Test if fatal errors are caught. Call an undefined function, foo().
++   */
++  function testKobayashiMaru() {
++    if (!function_exists('foo')) {
++      foo();
++    }
++    $this->fail();
++  }
+ 
+   /**
+    * Test the internal browsers functionality.