diff --git a/core/modules/media/src/MediaSourceBase.php b/core/modules/media/src/MediaSourceBase.php
index dea0140260..bc11213a73 100644
--- a/core/modules/media/src/MediaSourceBase.php
+++ b/core/modules/media/src/MediaSourceBase.php
@@ -317,4 +317,11 @@ protected function getSourceFieldName() {
     return $id;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getDisplayOptions(MediaTypeInterface $type, $display_context) {
+    return [];
+  }
+
 }
diff --git a/core/modules/media/src/MediaSourceInterface.php b/core/modules/media/src/MediaSourceInterface.php
index 723e0b9a8e..2c296da360 100644
--- a/core/modules/media/src/MediaSourceInterface.php
+++ b/core/modules/media/src/MediaSourceInterface.php
@@ -139,4 +139,20 @@ public function getSourceFieldDefinition(MediaTypeInterface $type);
    */
   public function createSourceField(MediaTypeInterface $type);
 
+  /**
+   * Gets the options for the source field in the form/view display.
+   *
+   * @param \Drupal\media\MediaTypeInterface $type
+   *   The media type which is using this source.
+   * @param string $display_context
+   *   The display context which will use these options. Can be 'view' or 'form.
+   *
+   * @return array|bool
+   *   The display options, or FALSE for removing this component from
+   *   the display.
+   *
+   * @see \Drupal\Core\Entity\Display\EntityDisplayInterface::setComponent()
+   */
+  public function getDisplayOptions(MediaTypeInterface $type, $display_context);
+
 }
diff --git a/core/modules/media/src/MediaTypeForm.php b/core/modules/media/src/MediaTypeForm.php
index 5f301d58ed..130056ef0d 100644
--- a/core/modules/media/src/MediaTypeForm.php
+++ b/core/modules/media/src/MediaTypeForm.php
@@ -4,6 +4,7 @@
 
 use Drupal\Core\Ajax\AjaxResponse;
 use Drupal\Core\Ajax\ReplaceCommand;
+use Drupal\Core\Entity\Display\EntityDisplayInterface;
 use Drupal\Core\Entity\EntityFieldManagerInterface;
 use Drupal\Core\Entity\EntityForm;
 use Drupal\Core\Field\BaseFieldDefinition;
@@ -328,29 +329,19 @@ public function save(array $form, FormStateInterface $form_state) {
       // Add the new field to the default form and view displays for this
       // media type.
       $field_name = $source_field->getName();
-      $field_type = $source_field->getType();
-
       if ($source_field->isDisplayConfigurable('form')) {
-        // Use the default widget and settings.
-        $component = \Drupal::service('plugin.manager.field.widget')
-          ->prepareConfiguration($field_type, []);
-
         // @todo Replace entity_get_form_display() when #2367933 is done.
         // https://www.drupal.org/node/2872159.
-        entity_get_form_display('media', $media_type->id(), 'default')
-          ->setComponent($field_name, $component)
-          ->save();
+        $display = entity_get_form_display('media', $media_type->id(), 'default');
+        $this->setDisplayOptions($display, $field_name, $source->getDisplayOptions($media_type, 'form'));
+        $display->save();
       }
       if ($source_field->isDisplayConfigurable('view')) {
-        // Use the default formatter and settings.
-        $component = \Drupal::service('plugin.manager.field.formatter')
-          ->prepareConfiguration($field_type, []);
-
         // @todo Replace entity_get_display() when #2367933 is done.
         // https://www.drupal.org/node/2872159.
-        entity_get_display('media', $media_type->id(), 'default')
-          ->setComponent($field_name, $component)
-          ->save();
+        $display = entity_get_display('media', $media_type->id(), 'default');
+        $this->setDisplayOptions($display, $field_name, $source->getDisplayOptions($media_type, 'view'));
+        $display->save();
       }
     }
 
@@ -375,4 +366,26 @@ public function save(array $form, FormStateInterface $form_state) {
     $form_state->setRedirectUrl($media_type->toUrl('collection'));
   }
 
+  /**
+   * Sets source display options on an entity display.
+   *
+   * @param \Drupal\Core\Entity\Display\EntityDisplayInterface $entity_display
+   *   A entity display.
+   * @param string $field_name
+   *   Field that will be configured in the display.
+   * @param array|bool $display_options
+   *   The source display options.
+   */
+  protected function setDisplayOptions(EntityDisplayInterface $entity_display, $field_name, $display_options) {
+    if ($display_options === FALSE) {
+      $entity_display->removeComponent($field_name);
+    }
+    elseif (is_array($display_options)) {
+      $entity_display->setComponent($field_name, $display_options);
+    }
+    else {
+      $entity_display->setComponent($field_name);
+    }
+  }
+
 }
diff --git a/core/modules/media/tests/modules/media_test_source/config/schema/media_test_source.schema.yml b/core/modules/media/tests/modules/media_test_source/config/schema/media_test_source.schema.yml
index 089aeac35f..ab025640b4 100644
--- a/core/modules/media/tests/modules/media_test_source/config/schema/media_test_source.schema.yml
+++ b/core/modules/media/tests/modules/media_test_source/config/schema/media_test_source.schema.yml
@@ -13,3 +13,11 @@ media.source.test_translation:
 media.source.test_constraints:
   type: media.source.test
   label: 'Test media source with constraints configuration'
+
+media.source.test_hidden_source_field:
+  type: media.source.test
+  label: 'Test media source with hidden source field'
+
+media.source.test_different_displays:
+  type: media.source.test
+  label: 'Test media source with different source field displays'
diff --git a/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestDifferentDisplays.php b/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestDifferentDisplays.php
new file mode 100644
index 0000000000..4f991807bb
--- /dev/null
+++ b/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestDifferentDisplays.php
@@ -0,0 +1,43 @@
+<?php
+
+namespace Drupal\media_test_source\Plugin\media\Source;
+
+use Drupal\media\MediaTypeInterface;
+
+/**
+ * Provides test media source.
+ *
+ * @MediaSource(
+ *   id = "test_different_displays",
+ *   label = @Translation("Test source with different displays"),
+ *   description = @Translation("Test source with different displays."),
+ *   allowed_field_types = {"entity_reference"},
+ * )
+ */
+class TestDifferentDisplays extends Test {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDisplayOptions(MediaTypeInterface $type, $display_context) {
+    switch ($display_context) {
+      case 'view':
+        return [
+          'type' => 'entity_reference_entity_id',
+        ];
+
+      case 'form':
+        return [
+          'type' => 'entity_reference_autocomplete_tags',
+        ];
+    }
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getSourceFieldName() {
+    return 'field_media_different_display';
+  }
+
+}
diff --git a/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestWithHiddenSourceField.php b/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestWithHiddenSourceField.php
new file mode 100644
index 0000000000..42640f990b
--- /dev/null
+++ b/core/modules/media/tests/modules/media_test_source/src/Plugin/media/Source/TestWithHiddenSourceField.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace Drupal\media_test_source\Plugin\media\Source;
+
+use Drupal\media\MediaTypeInterface;
+
+/**
+ * Provides test media source.
+ *
+ * @MediaSource(
+ *   id = "test_hidden_source_field",
+ *   label = @Translation("Test source with hidden source field"),
+ *   description = @Translation("Test media source with hidden source field."),
+ *   allowed_field_types = {"string"},
+ * )
+ */
+class TestWithHiddenSourceField extends Test {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDisplayOptions(MediaTypeInterface $type, $display_context) {
+    return FALSE;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getSourceFieldName() {
+    return 'field_media_hidden';
+  }
+
+}
diff --git a/core/modules/media/tests/src/Kernel/MediaSourceTest.php b/core/modules/media/tests/src/Kernel/MediaSourceTest.php
index fc30326fb1..c7f89a490e 100644
--- a/core/modules/media/tests/src/Kernel/MediaSourceTest.php
+++ b/core/modules/media/tests/src/Kernel/MediaSourceTest.php
@@ -442,4 +442,79 @@ public function testSourceConfigurationSubmit() {
     $this->assertEquals($expected, $source->getConfiguration(), 'Submitted values were saved correctly.');
   }
 
+  /**
+   * Tests different display options for the source field.
+   */
+  public function testDifferentSourceFieldDisplays() {
+    $id = 'test_different_displays';
+    $field_name = 'field_media_different_display';
+
+    $this->createMediaTypeViaForm($id, $field_name);
+
+    // Source field not in displays.
+    $display = entity_get_display('media', $id, 'default');
+    $components = $display->getComponents();
+    $this->assertArrayHasKey($field_name, $components);
+    $this->assertSame('entity_reference_entity_id', $components[$field_name]['type']);
+
+    $display = entity_get_form_display('media', $id, 'default');
+    $components = $display->getComponents();
+    $this->assertArrayHasKey($field_name, $components);
+    $this->assertSame('entity_reference_autocomplete_tags', $components[$field_name]['type']);
+  }
+
+  /**
+   * Tests hidden source field in media type.
+   */
+  public function testHiddenSourceField() {
+    $id = 'test_hidden_source_field';
+    $field_name = 'field_media_hidden';
+
+    $this->createMediaTypeViaForm($id, $field_name);
+
+    // Source field not in displays.
+    $display = entity_get_display('media', $id, 'default');
+    $this->assertArrayNotHasKey($field_name, $display->getComponents());
+
+    $display = entity_get_form_display('media', $id, 'default');
+    $this->assertArrayNotHasKey($field_name, $display->getComponents());
+  }
+
+  /**
+   * Creates a media type via form submit.
+   *
+   * @param int $source_plugin_id
+   *   Source plugin ID.
+   * @param string $field_name
+   *   Source field name.
+   */
+  protected function createMediaTypeViaForm($source_plugin_id, $field_name) {
+    /** @var \Drupal\media\MediaTypeInterface $type */
+    $type = MediaType::create(['source' => $source_plugin_id]);
+
+    $form = $this->container->get('entity_type.manager')
+      ->getFormObject('media_type', 'add')
+      ->setEntity($type);
+
+    $form_state = new FormState();
+    $form_state->setValues([
+      'label' => 'Test type',
+      'id' => $source_plugin_id,
+      'op' => t('Save'),
+    ]);
+
+    /** @var \Drupal\Core\Entity\EntityFieldManagerInterface $field_manager */
+    $field_manager = \Drupal::service('entity_field.manager');
+
+    // Source field not created yet.
+    $fields = $field_manager->getFieldDefinitions('media', $source_plugin_id);
+    $this->assertArrayNotHasKey($field_name, $fields);
+
+    \Drupal::formBuilder()->submitForm($form, $form_state);
+
+    // Source field exists now.
+    $fields = $field_manager->getFieldDefinitions('media', $source_plugin_id);
+    $this->assertArrayHasKey($field_name, $fields);
+  }
+
 }