From b03dc0bb947086ad112cc94e72650336a7306434 Mon Sep 17 00:00:00 2001 From: Andrew Berry Date: Mon, 16 Jan 2012 17:11:22 -0500 Subject: [PATCH 1/2] Issue #1407196: Backport of 7.x-1.x ping queue processing. --- includes/ooyala.pages.inc | 154 --------------------------- includes/ooyala.ping.inc | 253 +++++++++++++++++++++++++++++++++++++++++++++ ooyala.module | 12 ++- 3 files changed, 263 insertions(+), 156 deletions(-) create mode 100644 includes/ooyala.ping.inc diff --git a/includes/ooyala.pages.inc b/includes/ooyala.pages.inc index 770cc4c..c80489a 100644 --- a/includes/ooyala.pages.inc +++ b/includes/ooyala.pages.inc @@ -335,157 +335,3 @@ function ooyala_refresh_thumbnail() { drupal_json($data); } -/** - * Menu callback; Respond to Ooyala pings when a video has finished processing. - * - * This callback responds to the URL "ooyala/ping" within the Drupal site. - * Typically this URL needs to be configured to receive pings from Ooyala.com - * when a video has finished processing and is ready to be displayed. The - * configuration for setting up a ping URL is at - * https://www.ooyala.com/backlot/web, underneath the Account => Developers - * tab. Enter a URL in the API Ping URL such as http://example.com/ooyala/ping, - * if Drupal is installed at the base of the domain. - */ - -function ooyala_ping() { - module_load_include('inc', 'ooyala', 'includes/ooyala.api'); - $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); - - if (isset($_GET['embedCode'])) { - $embedcode = $_GET['embedCode']; - $embed_args = array('%embedcode' => $embedcode); - if ($verbose) { - watchdog('ooyala_ping', 'Received ping from Ooyala Backlot for embedCode %embedcode with request 
@request
.', $embed_args + array('@request' => print_r($_REQUEST, TRUE))); - } - - if ($video = ooyala_api_video_load($embedcode)) { - _ooyala_ping_fetch_image($embedcode, $embed_args); - - if ($nodes = ooyala_load_nodes($embedcode)) { - foreach ($nodes as $node) { - _ooyala_ping_autopublish($node, $embed_args); - _ooyala_ping_metadata($node, $video, $embedcode, $embed_args); - } - } - else { - watchdog('ooyala_ping', 'Content matching the embed code %embedcode was not found. No action has been taken.', $embed_args, WATCHDOG_WARNING); - } - } - else { - watchdog('ooyala_ping', 'A matching video for the %embedcode could not be loaded from Ooyala.', $embed_args, WATCHDOG_ERROR); - } - } - else { - if ($verbose) { - watchdog('ooyala_ping', 'Received embedcode-less ping from Ooyala Backlot with request 
@request
.', array('@request' => print_r($_REQUEST, TRUE))); - } - else { - watchdog('ooyala_ping', 'Recieved ping without an embed code. No action has been taken.', array(), WATCHDOG_ERROR); - } - } - - // Disable the page cache for this request. - if (function_exists('drupal_page_is_cacheable')) { - // PressFlow or Drupal 7 mechanism: - drupal_page_is_cacheable(FALSE); - } - else { - // Drupal 6 cache disabling hack. - $GLOBALS['conf']['cache'] = CACHE_DISABLED; - } -} - -/** - * Fetch an image in response to an Ooyala ping. - * - * @param $embedcode - * The embed code to fetch the image for. - * @param $embed_args - * An array of arguments for substition into translated watchdog messages. - * - * @return - * The image path of the fetched image, or FALSE if one could not be loaded. - */ -function _ooyala_ping_fetch_image($embedcode, $embed_args) { - $image_path = NULL; - if ($image_path = ooyala_api_fetch_image($embedcode)) { - if (variable_get('ooyala_ping_reporting_verbose', FALSE)) { - watchdog('ooyala_ping', 'The video image for %embedcode has been downloaded to @filename.', $embed_args + array('!url' => file_create_url($image_path), '@filename' => basename($image_path))); - } - } - else { - watchdog('ooyala_ping', 'There was an error downloading the video image for %embedcode.', $embed_args, WATCHDOG_ERROR); - } - - return $image_path; -} - -/** - * Mark a node as published if the video has been published. - * - * @param &$node - * The node object to mark as published. - * @param $embed_args - * An array of arguments for substition into translated watchdog messages. - */ -function _ooyala_ping_autopublish(&$node, $embed_args) { - $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); - if (variable_get('ooyala_autopublish', 0) && !$node->status) { - $node->status = 1; - node_save($node); - watchdog('ooyala_ping', 'The content %title has been published by a ping for %embedcode.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title)); - } - elseif (!variable_get('ooyala_autopublish', 0)) { - if ($verbose) { - watchdog('ooyala_ping', 'The content %title with embed code %embedcode was not published as automatic publishing is disabled.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title)); - } - } - else { - if ($verbose) { - watchdog('ooyala_ping', 'The content %title with embed code %embedcode was already published.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title), WATCHDOG_WARNING); - } - } -} - -/** - * Update the saved metadata in a node's video field. - * - * @param &$node - * The node object to mark as published. - * @param $video - * The video array as loaded from the Ooyala API. - * @param $embedcode - * The embed code to fetch the image for. - * @param $embed_args - * An array of arguments for substition into translated watchdog messages. - */ -function _ooyala_ping_metadata(&$node, $video, $embedcode, $embed_args) { - $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); - foreach ($node->ooyala_field_names as $field_name) { - foreach ($node->{$field_name} as $key => $value) { - if ($value['value'] == $embedcode) { - // Update the meta data in the node if changed. - if (empty($value['status']) || $value['status'] != $video['status']) { - $node->{$field_name}[$key]['status'] = $video['status']; - $node->pulled_from_ooyala = TRUE; - } - if (empty($value['length']) || $value['length'] != $video['length']) { - $node->{$field_name}[$key]['length'] = $video['length']; - $node->pulled_from_ooyala = TRUE; - } - } - } - } - if (!empty($node->pulled_from_ooyala)) { - if ($verbose) { - watchdog('ooyala_ping', 'Updated video %embedcode status (@status) and length (@length) values.', $embed_args + array('@status' => $video['status'], '@length' => $video['length'])); - } - node_save($node); - } - else { - if ($verbose) { - watchdog('ooyala_ping', 'No changes necessary to video status or length for embed code %embedcode.', $embed_args); - } - } -} - diff --git a/includes/ooyala.ping.inc b/includes/ooyala.ping.inc new file mode 100644 index 0000000..cea0d36 --- /dev/null +++ b/includes/ooyala.ping.inc @@ -0,0 +1,253 @@ + Developers + * tab. Enter a URL in the API Ping URL such as http://example.com/ooyala/ping, + * if Drupal is installed at the base of the domain. + */ +function ooyala_ping_insert() { + // Disable the page cache for this request. + if (function_exists('drupal_page_is_cacheable')) { + // PressFlow or Drupal 7 mechanism: + drupal_page_is_cacheable(FALSE); + } + else { + // Drupal 6 cache disabling hack. + $GLOBALS['conf']['cache'] = CACHE_DISABLED; + } + + $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); + + if (isset($_GET['embedCode'])) { + $data = array( + 'embed_code' => $_GET['embedCode'], + 'fetch_attempts' => 0, + ); + + if ($verbose) { + watchdog('ooyala_ping', 'Received ping from Ooyala Backlot for embedCode %embed_code with request 
@request
.', array('%embed_code' => $_GET['embedCode'], '@request' => print_r($_REQUEST, TRUE))); + } + + $queue = ooyala_ping_queue(); + $queue->createItem($data); + } + else { + if ($verbose) { + watchdog('ooyala_ping', 'Received embedcode-less ping from Ooyala Backlot with request 
@request
.', array('@request' => print_r($_REQUEST, TRUE))); + } + else { + watchdog('ooyala_ping', 'Recieved ping without an embed code. No action has been taken.', array(), WATCHDOG_ERROR); + } + } +} + +/** + * Return the Ooyala Ping queue, creating it if needed. + * + * @return + * An instance of the default queuing mechanism used for storing Ooyala pings + * to process. + */ +function ooyala_ping_queue() { + $queue = DrupalQueue::get('ooyala_ping'); + $queue->createQueue(); + return $queue; +} + +/** + * Process items in the Ooyala ping queue. + * + * @param $limit + * Optional parameter indicating the number of items to process at once. + */ +function ooyala_ping_queue_process($limit = NULL) { + $queue = ooyala_ping_queue(); + + if (!$limit) { + $limit = 10; + } + + // Limit to at most the number of items in the queue. This prevents us from + // repeatedly fetching the same embed code during the same processing phase + // if the embed code is encountering errors processing. + $limit = min($limit, $queue->numberOfItems()); + + // Keep an array of all embed codes processed during this request. We use + // this to prevent multiple API calls where the same embed code has been + // pinged in beteween queue runs. We don't worry about uniqueness across + // multiple requests to save us from having to handle any potential race + // conditions where another API call is actually needed. + $processed = array(); + + while (count($processed) < $limit && $item = $queue->claimItem()) { + $data = $item->data; + $embed_code = $data['embed_code']; + + if (!in_array($embed_code, $processed)) { + $processed[] = $embed_code; + + if (!ooyala_ping_process($data['embed_code'])) { + $data['fetch_attempts']++; + if ($data['fetch_attempts'] < variable_get('ooyala_ping_fetch_attempts', 10)) { + $queue->createItem($data); + } + else { + watchdog('ooyala_ping', 'Unable to process ping request for embed code %embed_code after !count attempts. Discarding the embed code from the queue.', array('%embed_code' => $data['embed_code'], '!count' => $data['fetch_attempts']), WATCHDOG_ERROR); + } + } + } + $queue->deleteItem($item); + } +} + +/** + * Process an embed code that has been queued. + * + * @param $embed_code + * The embed code to process by syncing data such as video metadata and node + * statuses. + * + * @return + * TRUE if the embed code was processed successfully, or FALSE otherwise. + */ +function ooyala_ping_process($embed_code) { + module_load_include('inc', 'ooyala', 'includes/ooyala.api'); + $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); + + $embed_args = array('%embed_code' => $embed_code); + + if ($video = ooyala_api_video_load($embed_code)) { + // If we can't fetch an image for a video, odds are the rest of the + // request will fail too. So, we bail here and don't publish the node so + // anything dependent on the thumbnail image (such as views) won't break. + if (!_ooyala_ping_fetch_image($embed_code, $embed_args)) { + return FALSE; + } + + if ($nodes = ooyala_load_nodes($embed_code)) { + foreach ($nodes as $node) { + _ooyala_ping_autopublish($node, $embed_args); + _ooyala_ping_metadata($node, $video, $embed_code, $embed_args); + } + } + else { + watchdog('ooyala_ping', 'Content matching the embed code %embed_code was not found. No action has been taken.', $embed_args, WATCHDOG_WARNING); + return FALSE; + } + } + else { + watchdog('ooyala_ping', 'A matching video for the %embed_code could not be loaded from Ooyala.', $embed_args, WATCHDOG_ERROR); + return FALSE; + } + + return TRUE; +} + +/** + * Fetch an image in response to an Ooyala ping. + * + * @param $embed_code + * The embed code to fetch the image for. + * @param $embed_args + * An array of arguments for substition into translated watchdog messages. + * + * @return + * The image path of the fetched image, or FALSE if one could not be loaded. + */ +function _ooyala_ping_fetch_image($embed_code, $embed_args) { + $image_path = NULL; + if ($image_path = ooyala_api_fetch_image($embed_code)) { + if (variable_get('ooyala_ping_reporting_verbose', FALSE)) { + watchdog('ooyala_ping', 'The video image for %embed_code has been downloaded to @filename.', $embed_args + array('!url' => file_create_url($image_path), '@filename' => basename($image_path))); + } + } + else { + watchdog('ooyala_ping', 'There was an error downloading the video image for %embed_code.', $embed_args, WATCHDOG_ERROR); + return FALSE; + } + + return $image_path; +} + +/** + * Mark a node as published if the video has been published. + * + * @param &$node + * The node object to mark as published. + * @param $embed_args + * An array of arguments for substition into translated watchdog messages. + */ +function _ooyala_ping_autopublish(&$node, $embed_args) { + $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); + if (variable_get('ooyala_autopublish', 0) && !$node->status) { + $node->status = 1; + node_save($node); + watchdog('ooyala_ping', 'The content %title has been published by a ping for %embed_code.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title)); + } + elseif (!variable_get('ooyala_autopublish', 0)) { + if ($verbose) { + watchdog('ooyala_ping', 'The content %title with embed code %embed_code was not published as automatic publishing is disabled.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title)); + } + } + else { + if ($verbose) { + watchdog('ooyala_ping', 'The content %title with embed code %embed_code was already published.', $embed_args + array('!url' => url('node/' . $node->nid), '%title' => $node->title), WATCHDOG_WARNING); + } + } +} + +/** + * Update the saved metadata in a node's video field. + * + * @param &$node + * The node object to mark as published. + * @param $video + * The video array as loaded from the Ooyala API. + * @param $embed_code + * The embed code to fetch the image for. + * @param $embed_args + * An array of arguments for substition into translated watchdog messages. + */ +function _ooyala_ping_metadata(&$node, $video, $embed_code, $embed_args) { + $verbose = variable_get('ooyala_ping_reporting_verbose', FALSE); + foreach ($node->ooyala_field_names as $field_name) { + foreach ($node->{$field_name} as $key => $value) { + if ($value['value'] == $embed_code) { + // Update the meta data in the node if changed. + if (empty($value['status']) || $value['status'] != $video['status']) { + $node->{$field_name}[$key]['status'] = $video['status']; + $node->pulled_from_ooyala = TRUE; + } + if (empty($value['length']) || $value['length'] != $video['length']) { + $node->{$field_name}[$key]['length'] = $video['length']; + $node->pulled_from_ooyala = TRUE; + } + } + } + } + if (!empty($node->pulled_from_ooyala)) { + if ($verbose) { + watchdog('ooyala_ping', 'Updated video %embed_code status (@status) and length (@length) values.', $embed_args + array('@status' => $video['status'], '@length' => $video['length'])); + } + node_save($node); + } + else { + if ($verbose) { + watchdog('ooyala_ping', 'No changes necessary to video status or length for embed code %embed_code.', $embed_args); + } + } +} + diff --git a/ooyala.module b/ooyala.module index cd8ec11..0b13254 100755 --- a/ooyala.module +++ b/ooyala.module @@ -51,9 +51,9 @@ function ooyala_menu() { 'type' => MENU_CALLBACK, ); $items['ooyala/ping'] = array( - 'page callback' => 'ooyala_ping', + 'page callback' => 'ooyala_ping_insert', 'access arguments' => array('access content'), - 'file' => 'includes/ooyala.pages.inc', + 'file' => 'includes/ooyala.ping.inc', 'type' => MENU_CALLBACK, ); @@ -236,6 +236,14 @@ function ooyala_cron() { // Set the sync time so that we don't need to process unchanged videos. variable_set('ooyala_last_sync', $sync_time); + + // Fire off the processing of queued embed codes. Note that we manually run + // the queue here instead of using hook_queue_cron_info(). If we use that + // hook, there is no easy way to limit requests sent per cron run, not giving + // us enough time to allow the network or Ooyala to return in the case of a + // failure. + module_load_include('inc', 'ooyala', 'includes/ooyala.ping'); + ooyala_ping_queue_process(variable_get('ooyala_ping_queue_limit', 10)); } /** -- 1.7.7.4 From ee524a42d3fa7e148505d3b7eb5e34b20e8f8105 Mon Sep 17 00:00:00 2001 From: Andrew Berry Date: Tue, 31 Jan 2012 11:58:28 -0500 Subject: [PATCH 2/2] Issue #1407196: Require the drupal_queue module and include it's files when needed. --- includes/ooyala.ping.inc | 1 + ooyala.info | 2 ++ 2 files changed, 3 insertions(+), 0 deletions(-) diff --git a/includes/ooyala.ping.inc b/includes/ooyala.ping.inc index cea0d36..7e2617a 100644 --- a/includes/ooyala.ping.inc +++ b/includes/ooyala.ping.inc @@ -62,6 +62,7 @@ function ooyala_ping_insert() { * to process. */ function ooyala_ping_queue() { + drupal_queue_include(); $queue = DrupalQueue::get('ooyala_ping'); $queue->createQueue(); return $queue; diff --git a/ooyala.info b/ooyala.info index 977703a..835efc0 100755 --- a/ooyala.info +++ b/ooyala.info @@ -3,3 +3,5 @@ description = Allows uploading and display of videos from the