Index: includes/form.inc
===================================================================
RCS file: /cvs/drupal/drupal/includes/form.inc,v
retrieving revision 1.408
diff -u -p -r1.408 form.inc
--- includes/form.inc	28 Nov 2009 14:39:31 -0000	1.408
+++ includes/form.inc	30 Nov 2009 06:29:58 -0000
@@ -333,11 +333,6 @@ function drupal_rebuild_form($form_id, &
     form_set_cache($form_build_id, $form, $form_state);
   }
 
-  // Clear out all post data, as we don't want the previous step's
-  // data to pollute this one and trigger validate/submit handling,
-  // then process the form for rendering.
-  $form_state['input'] = array();
-
   // Also clear out all group associations as these might be different
   // when rerendering the form.
   $form_state['groups'] = array();
@@ -414,7 +409,10 @@ function form_set_cache($form_build_id, 
  *   A keyed array containing the current state of the form. Most
  *   important is the $form_state['values'] collection, a tree of data
  *   used to simulate the incoming $_POST information from a user's
- *   form submission.
+ *   form submission. If a key is not filled in $form_state['values'] or NULL
+ *   then the default value of the respective element is used. For a checkbox
+ *   element in $_POST this would mean uncheck so to uncheck a checkbox, literal
+ *   FALSE must be set in $form_state['values'].
  * @param ...
  *   Any additional arguments are passed on to the functions called by
  *   drupal_form_submit(), including the unique form constructor function.
@@ -937,9 +935,11 @@ function _form_validate(&$elements, &$fo
  */
 function form_execute_handlers($type, &$form, &$form_state) {
   $return = FALSE;
+  // If there was a button pressed, use its handlers.
   if (isset($form_state[$type . '_handlers'])) {
     $handlers = $form_state[$type . '_handlers'];
   }
+  // Otherwise, check for a form level handler.
   elseif (isset($form['#' . $type])) {
     $handlers = $form['#' . $type];
   }
@@ -1211,9 +1211,35 @@ function _form_builder_handle_input_elem
       foreach ($element['#parents'] as $parent) {
         $input = isset($input[$parent]) ? $input[$parent] : NULL;
       }
-      // If we have input for the current element, assign it to the #value property.
-      if (!$form_state['programmed'] || isset($input)) {
-        // Call #type_value to set the form value;
+      // Browsers do not submit $_POST entries for unchecked checkboxes,
+      // multiple select fields without anything selected, and perhaps other
+      // controls. For these controls, we want to replace a browser submitted
+      // $input=NULL with $input=FALSE, so that NULL can mean to use the
+      // element's default value, which is necessary during a rebuild (a rebuild
+      // that adds new elements will have $input=NULL for those elements) and
+      // convenient for programmatically submitted forms. This works, because
+      // $_POST data can't contain FALSE (only strings and arrays), making
+      // FALSE a reasonable internal substitute for intentionally NULL input.
+      if (!isset($input) && empty($form_state['rebuild']) && empty($form_state['programmed'])) {
+        $input = FALSE;
+        // In addition to converting $input, update $form_state['input'], so
+        // that if this form is rebuilt, it will be known what was submitted as
+        // intentionally null vs. what's new in the rebuilt form.
+        $ref = &$form_state['input'];
+        foreach ($element['#parents'] as $parent) {
+          $ref = &$ref[$parent];
+        }
+        $ref = FALSE;
+        // It's too fragile to keep references around.
+        unset($ref);
+      }
+      if (isset($input)) {
+        // Once here, we know we have intentional input, and can convert the
+        // local $input variable from FALSE back to NULL, because that's a more
+        // logical representation of nothing. 
+        if ($input === FALSE) {
+          $input = NULL;
+        }
         if (function_exists($value_callback)) {
           $element['#value'] = $value_callback($element, $input, $form_state);
         }
Index: modules/field/field.form.inc
===================================================================
RCS file: /cvs/drupal/drupal/modules/field/field.form.inc,v
retrieving revision 1.35
diff -u -p -r1.35 field.form.inc
--- modules/field/field.form.inc	20 Nov 2009 04:51:27 -0000	1.35
+++ modules/field/field.form.inc	30 Nov 2009 06:29:58 -0000
@@ -131,6 +131,11 @@ function field_default_form($obj_type, $
  * - drag-n-drop value reordering
  */
 function field_multiple_value_form($field, $instance, $langcode, $items, &$form, &$form_state) {
+  // This form has its own multistep persistance.
+  if (!empty($form_state['rebuild'])) {
+    $form_state['input'] = array();
+  }
+
   $field_name = $field['field_name'];
 
   // Determine the number of widgets to display.
Index: modules/node/node.pages.inc
===================================================================
RCS file: /cvs/drupal/drupal/modules/node/node.pages.inc,v
retrieving revision 1.100
diff -u -p -r1.100 node.pages.inc
--- modules/node/node.pages.inc	8 Nov 2009 10:02:41 -0000	1.100
+++ modules/node/node.pages.inc	30 Nov 2009 06:29:59 -0000
@@ -109,8 +109,13 @@ function node_object_prepare($node) {
  */
 function node_form($form, &$form_state, $node) {
   global $user;
+  // This form has its own multistep persistance.
+  if (!empty($form_state['rebuild'])) {
+    $form_state['input'] = array();
+  }
 
   if (isset($form_state['node'])) {
+    // Node has its own multistep persistance.
     $node = (object)($form_state['node'] + (array)$node);
   }
   if (isset($form_state['node_preview'])) {
Index: modules/simpletest/tests/ajax.test
===================================================================
RCS file: /cvs/drupal/drupal/modules/simpletest/tests/ajax.test,v
retrieving revision 1.3
diff -u -p -r1.3 ajax.test
--- modules/simpletest/tests/ajax.test	22 Nov 2009 02:48:37 -0000	1.3
+++ modules/simpletest/tests/ajax.test	30 Nov 2009 06:29:59 -0000
@@ -141,3 +141,48 @@ class AJAXCommandsTestCase extends AJAXT
     $this->assertTrue($command['command'] == 'restripe' && $command['selector'] == '#restripe_table', "'restripe' AJAX command issued with correct selector");
   }
 }
+
+/**
+ * Test that $form_state['values'] is properly delivered to $ajax['callback'].
+ */
+class AJAXFormValuesTestCase extends AJAXTestCase {
+  public static function getInfo() {
+    return array(
+      'name' => 'AJAX command form values',
+      'description' => 'Tests that form values are properly delivered to AJAX callbacks.',
+      'group' => 'AJAX',
+    );
+  }
+
+  function setUp() {
+    parent::setUp();
+
+    $this->web_user = $this->drupalCreateUser(array('access content'));
+    $this->drupalLogin($this->web_user);
+  }
+
+  /**
+   * Create a simple form, then POST to system/ajax to change to it.
+   */
+  function testSimpleAJAXFormValue() {
+    // Verify form values of a select element.
+    foreach(array('red', 'green', 'blue') as $item) {
+      $edit = array(
+        'select' => $item,
+      );
+      $commands = $this->drupalPostAJAX('ajax_forms_test_get_form', $edit, 'select');
+      $data_command = $commands[2];
+      $this->assertEqual($data_command['value'], $item);
+    }
+
+    // Verify form values of a checkbox element.
+    foreach(array(FALSE, TRUE) as $item) {
+      $edit = array(
+        'checkbox' => $item,
+      );
+      $commands = $this->drupalPostAJAX('ajax_forms_test_get_form', $edit, 'checkbox');
+      $data_command = $commands[2];
+      $this->assertEqual((int) $data_command['value'], (int) $item);
+    }
+  }
+}
Index: modules/simpletest/tests/form.test
===================================================================
RCS file: /cvs/drupal/drupal/modules/simpletest/tests/form.test,v
retrieving revision 1.26
diff -u -p -r1.26 form.test
--- modules/simpletest/tests/form.test	28 Nov 2009 14:39:31 -0000	1.26
+++ modules/simpletest/tests/form.test	30 Nov 2009 06:29:59 -0000
@@ -131,9 +131,7 @@ class FormsTestCase extends DrupalWebTes
     $this->assertRaw(t('!name field is required.', array('!name' => 'required_checkbox')), t('A required checkbox is actually mandatory'));
 
     // Now try to submit the form correctly.
-    $this->drupalPost(NULL, array('required_checkbox' => 1), t('Submit'));
-
-    $values = json_decode($this->drupalGetContent(), TRUE);
+    $values = drupal_json_decode($this->drupalPost(NULL, array('required_checkbox' => 1), t('Submit')));
     $expected_values = array(
       'disabled_checkbox_on' => 'disabled_checkbox_on',
       'disabled_checkbox_off' => '',
@@ -468,16 +466,15 @@ class FormsFormStorageTestCase extends D
 
   function setUp() {
     parent::setUp('form_test');
+
+    $this->web_user = $this->drupalCreateUser(array('access content'));
+    $this->drupalLogin($this->web_user);
   }
 
   /**
    * Tests using the form in a usual way.
    */
   function testForm() {
-
-    $user = $this->drupalCreateUser(array('access content'));
-    $this->drupalLogin($user);
-
     $this->drupalPost('form_test/form-storage', array('title' => 'new', 'value' => 'value_is_set'), 'Continue');
     $this->assertText('Form constructions: 2', t('The form has been constructed two times till now.'));
 
@@ -490,9 +487,6 @@ class FormsFormStorageTestCase extends D
    * Tests using the form with an activated $form_state['cache'] property.
    */
   function testFormCached() {
-    $user = $this->drupalCreateUser(array('access content'));
-    $this->drupalLogin($user);
-
     $this->drupalPost('form_test/form-storage', array('title' => 'new', 'value' => 'value_is_set'), 'Continue', array('query' => array('cache' => 1)));
     $this->assertText('Form constructions: 1', t('The form has been constructed one time till now.'));
 
@@ -505,12 +499,54 @@ class FormsFormStorageTestCase extends D
    * Tests validation when form storage is used.
    */
   function testValidation() {
-    $user = $this->drupalCreateUser(array('access content'));
-    $this->drupalLogin($user);
-
     $this->drupalPost('form_test/form-storage', array('title' => '', 'value' => 'value_is_set'), 'Continue');
     $this->assertPattern('/value_is_set/', t("The input values have been kept."));
   }
+
+  /**
+   * Tests updating cached form storage during form validation.
+   *
+   * If form caching is enabled and a form stores data in the form storage, then
+   * the form storage also has to be updated in case of a validation error in
+   * the form. This test re-uses the existing form for multi-step tests, but
+   * triggers a special #element_validate handler to update the form storage
+   * during form validation, while another, required element in the form
+   * triggers a form validation error.
+   */
+  function testCachedFormStorageValidation() {
+    // Request the form with 'cache' query parameter to enable form caching.
+    $this->drupalGet('form_test/form-storage', array('query' => array('cache' => 1)));
+    // Also retrieve the form_build_id to assert that it keeps identical
+    // throughout this entire test.
+    $fields = $this->xpath($this->constructFieldXpath('name', 'form_build_id'));
+    $form_build_id = (string) $fields[0]['value'];
+
+    // Skip step 1 of the multi-step form, since the first step copies over
+    // 'title' into form storage, but we want to verify that changes in the form
+    // storage are updated in the cache during form validation.
+    $edit = array('title' => 'foo');
+    $this->drupalPost(NULL, $edit, 'Continue');
+    #$this->assertFieldByName('form_build_id', $form_build_id);
+
+    // In step 2, trigger a validation error for the required 'title' field, and
+    // post the special 'change_title' value for the 'value' field, which
+    // conditionally invokes the #element_validate handler to update the form
+    // storage.
+    $edit = array('title' => '', 'value' => 'change_title');
+    $this->drupalPost(NULL, $edit, 'Save');
+    #$this->assertFieldByName('form_build_id', $form_build_id);
+
+    // At this point, the form storage should contain updated values, but we do
+    // not see them, because the form has not been rebuilt yet due to the
+    // validation error. Post again with an arbitrary 'title' (which is only
+    // updated in form storage in step 1) and verify that the rebuilt form
+    // contains the values of the updated form storage.
+    $edit = array('title' => 'foo', 'value' => '');
+    $this->drupalPost(NULL, $edit, 'Save');
+    #$this->assertFieldByName('form_build_id', $form_build_id);
+    $this->assertFieldByName('title', 'title_changed', t('The altered form storage value was updated in cache and taken over.'));
+    $this->assertText('Title: title_changed', t('The form storage has stored the values.'));
+  }
 }
 
 /**
@@ -559,8 +595,7 @@ class FormStateValuesCleanTestCase exten
    * Tests form_state_values_clean().
    */
   function testFormStateValuesClean() {
-    $this->drupalPost('form_test/form-state-values-clean', array(), t('Submit'));
-    $values = json_decode($this->content, TRUE);
+    $values = drupal_json_decode($this->drupalPost('form_test/form-state-values-clean', array(), t('Submit')));
 
     // Setup the expected result.
     $result = array(
@@ -588,3 +623,46 @@ class FormStateValuesCleanTestCase exten
   }
 }
 
+/**
+ * Tests form rebuilding.
+ *
+ * @todo Add tests for other aspects of form rebuilding.
+ */
+class FormsRebuildTestCase extends DrupalWebTestCase {
+  public static function getInfo() {
+    return array(
+      'name' => 'Form rebuilding',
+      'description' => 'Tests functionality of drupal_rebuild_form().',
+      'group' => 'Form API',
+    );
+  }
+
+  function setUp() {
+    parent::setUp('form_test');
+
+    $this->web_user = $this->drupalCreateUser(array('access content'));
+    $this->drupalLogin($this->web_user);
+  }
+
+  /**
+   * Tests preservation of values.
+   */
+  function testRebuildPreservesValues() {
+    $edit = array(
+      'checkbox_1_default_off' => TRUE,
+      'checkbox_1_default_on' => FALSE,
+      'text_1' => 'foo',
+    );
+    $this->drupalPost('form-test/form-rebuild-preserve-values', $edit, 'Add more');
+
+    // Verify that initial elements retained their submitted values.
+    $this->assertFieldChecked('edit-checkbox-1-default-off', t('A submitted checked checkbox retained its checked state during a rebuild.'));
+    $this->assertNoFieldChecked('edit-checkbox-1-default-on', t('A submitted unchecked checkbox retained its unchecked state during a rebuild.'));
+    $this->assertFieldById('edit-text-1', 'foo', t('A textfield retained its submitted value during a rebuild.'));
+
+    // Verify that newly added elements were initialized with their default values.
+    $this->assertFieldChecked('edit-checkbox-2-default-on', t('A newly added checkbox was initialized with a default checked state.'));
+    $this->assertNoFieldChecked('edit-checkbox-2-default-off', t('A newly added checkbox was initialized with a default unchecked state.'));
+    $this->assertFieldById('edit-text-2', 'DEFAULT 2', t('A newly added textfield was initialized with its default value.'));
+  }
+}
Index: modules/simpletest/tests/form_test.module
===================================================================
RCS file: /cvs/drupal/drupal/modules/simpletest/tests/form_test.module,v
retrieving revision 1.18
diff -u -p -r1.18 form_test.module
--- modules/simpletest/tests/form_test.module	28 Nov 2009 14:39:31 -0000	1.18
+++ modules/simpletest/tests/form_test.module	30 Nov 2009 06:29:59 -0000
@@ -57,7 +57,7 @@ function form_test_menu() {
   $items['form_test/form-storage'] = array(
     'title' => 'Form storage test',
     'page callback' => 'drupal_get_form',
-    'page arguments' => array('form_storage_test_form'),
+    'page arguments' => array('form_test_storage_form'),
     'access arguments' => array('access content'),
     'type' => MENU_CALLBACK,
   );
@@ -86,6 +86,14 @@ function form_test_menu() {
     'type' => MENU_CALLBACK,
   );
 
+  $items['form-test/form-rebuild-preserve-values'] = array(
+    'title' => 'Form values preservation during rebuild test',
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('form_test_form_rebuild_preserve_values_form'),
+    'access arguments' => array('access content'),
+    'type' => MENU_CALLBACK,
+  );
+
   return $items;
 }
 
@@ -370,9 +378,12 @@ function form_test_mock_form_submit($for
  * request parameter "cache" the form can be tested with caching enabled, as
  * it would be the case, if the form would contain some #ajax callbacks.
  *
- * @see form_storage_test_form_submit().
+ * @see form_test_storage_form_submit().
  */
-function form_storage_test_form($form, &$form_state) {
+function form_test_storage_form($form, &$form_state) {
+  if (!empty($form_state['rebuild'])) {
+    $form_state['input'] = array();
+  }
   // Initialize
   if (empty($form_state['storage'])) {
     if (empty($form_state['input'])) {
@@ -391,25 +402,29 @@ function form_storage_test_form($form, &
   // Count how often the form is constructed
   $_SESSION['constructions']++;
 
+  $form['title'] = array(
+    '#type' => 'textfield',
+    '#title' => 'Title',
+    '#default_value' => $form_state['storage']['thing']['title'],
+    '#required' => TRUE,
+  );
+  $form['value'] = array(
+    '#type' => 'textfield',
+    '#title' => 'Value',
+    '#default_value' => $form_state['storage']['thing']['value'],
+    '#element_validate' => array('form_test_storage_element_validate_value_cached'),
+  );
   if ($form_state['storage']['step'] == 1) {
-    $form['title'] = array(
-      '#type' => 'textfield',
-      '#title' => 'title',
-      '#default_value' => $form_state['storage']['thing']['title'],
-      '#required' => TRUE,
-    );
-    $form['value'] = array(
-      '#type' => 'textfield',
-      '#title' => 'value',
-      '#default_value' => $form_state['storage']['thing']['value'],
-    );
     $form['submit'] = array(
       '#type' => 'submit',
       '#value' => 'Continue',
     );
   }
   else {
-    $form['body'] = array('#value' => 'This is the second step.');
+    $form['body'] = array(
+      '#type' => 'item',
+      '#value' => 'This is the second step.',
+    );
     $form['submit'] = array(
       '#type' => 'submit',
       '#value' => 'Save',
@@ -426,9 +441,27 @@ function form_storage_test_form($form, &
 }
 
 /**
- * Multistep form submit callback.
+ * Form element validation handler for 'value' element in form_test_storage_form().
+ *
+ * Tests updating of cached form storage during validation.
+ */
+function form_test_storage_element_validate_value_cached($element, &$form_state) {
+  // If caching is enabled and we receive a certain value, change the value of
+  // 'title'. This presumes that another submitted form value triggers a
+  // validation error elsewhere in the form. Form API should still update the
+  // cached form storage though.
+  if (isset($_REQUEST['cache']) && $form_state['values']['value'] == 'change_title') {
+    $form_state['storage']['thing']['title'] = 'title_changed';
+    // @todo This really shouldn't be necessary. Should Form API cache that a
+    //   form can be cached? See http://drupal.org/node/641356
+    $form_state['cache'] = TRUE;
+  }
+}
+
+/**
+ * Form submit handler for form_test_storage_form().
  */
-function form_storage_test_form_submit($form, &$form_state) {
+function form_test_storage_form_submit($form, &$form_state) {
   if ($form_state['storage']['step'] == 1) {
     $form_state['storage']['thing']['title'] = $form_state['values']['title'];
     $form_state['storage']['thing']['value'] = $form_state['values']['value'];
@@ -567,3 +600,81 @@ function _form_test_checkbox_submit($for
   drupal_json_output($form_state['values']);
   exit();
 }
+
+/**
+ * Form builder for testing preservation of values during a rebuild.
+ */
+function form_test_form_rebuild_preserve_values_form($form, &$form_state) {
+  // Start the form with two checkboxes, to test different defaults, and a
+  // textfield, to test more than one element type.
+  $form = array(
+    'checkbox_1_default_off' => array(
+      '#type' => 'checkbox',
+      '#title' => t('This checkbox defaults to unchecked.'),
+      '#default_value' => FALSE,
+    ),
+    'checkbox_1_default_on' => array(
+      '#type' => 'checkbox',
+      '#title' => t('This checkbox defaults to checked.'),
+      '#default_value' => TRUE,
+    ),
+    'text_1' => array(
+      '#type' => 'textfield',
+      '#title' => t('This textfield has a non-empty default value.'),
+      '#default_value' => 'DEFAULT 1',
+    ),
+  );
+  // Provide an 'add more' button that rebuilds the form with an additional two
+  // checkboxes and a textfield. The test is to make sure that the rebuild
+  // triggered by this button preserves the user input values for the initial
+  // elements and initializes the new elements with the correct default values.
+  if (empty($form_state['storage']['add_more'])) {
+    $form['add_more'] = array(
+      '#type' => 'submit',
+      '#value' => 'Add more',
+      '#submit' => array('form_test_form_rebuild_preserve_values_form_add_more'),
+    );
+  }
+  else {
+    $form += array(
+      'checkbox_2_default_off' => array(
+        '#type' => 'checkbox',
+        '#title' => t('This checkbox defaults to unchecked.'),
+        '#default_value' => FALSE,
+      ),
+      'checkbox_2_default_on' => array(
+        '#type' => 'checkbox',
+        '#title' => t('This checkbox defaults to checked.'),
+        '#default_value' => TRUE,
+      ),
+      'text_2' => array(
+        '#type' => 'textfield',
+        '#title' => t('This textfield has a non-empty default value.'),
+        '#default_value' => 'DEFAULT 2',
+      ),
+    );
+  }
+  // A submit button that finishes the form workflow (does not rebuild).
+  $form['submit'] = array(
+    '#type' => 'submit',
+    '#value' => 'Submit',
+  );
+  return $form;
+}
+
+/**
+ * Button submit handler for form_test_form_rebuild_preserve_values_form().
+ */
+function form_test_form_rebuild_preserve_values_form_add_more($form, &$form_state) {
+  // Rebuild, to test preservation of input values.
+  $form_state['storage']['add_more'] = TRUE;
+  $form_state['rebuild'] = TRUE;
+}
+
+/**
+ * Form submit handler for form_test_form_rebuild_preserve_values_form().
+ */
+function form_test_form_rebuild_preserve_values_form_submit($form, &$form_state) {
+  // Finish the workflow. Do not rebuild.
+  drupal_set_message(t('Form values: %values', array('%values' => var_export($form_state['values'], TRUE))));
+}