=== added file 'includes/feed.inc' --- /dev/null +++ includes/feed.inc @@ -0,0 +1,363 @@ +code) { + case 304: // not modified + return array(); // return an empty set of items + break; + + case 301: // redirect + // update the URL and parse the data + $url = $feed->redirect_url; + + case 200: // all good, process + case 302: + case 307: + $feed_array = _feed_parse_feed($url, $feed->data); + if(!$feed_array) { + return NULL; + } + + if ($feed->headers['Last-Modified']) { + $feed_array['HEADERS']['modified'] = strtotime($result->headers['Last-Modified']); + } + if ($feed->headers['ETag']) { + $feed_array['HEADERS']['ETag'] = $feed->headersp['ETag']; + } + + return $feed_array; + default: + // todo: return something useful? + return NULL; + } + + // we should never get here... +} + +/** + * The interal feed parser. Not be called by modules. + */ +function _feed_parse_feed($url, $data) { + global $items, $image, $channel; + + // Unset the global variables before we use them: + unset($GLOBALS['element'], $GLOBALS['item'], $GLOBALS['tag']); + $items = array(); + $image = array(); + $channel = array(); + + // parse the data: + $xml_parser = drupal_xml_parser_create($data); + xml_set_element_handler($xml_parser, '_feed_element_start', '_feed_element_end'); + xml_set_character_data_handler($xml_parser, '_feed_element_data'); + + if (!xml_parse($xml_parser, $data, 1)) { + return NULL; + } + xml_parser_free($xml_parser); + + /* + ** Prepare the channel data: + */ + + foreach ($channel as $key => $value) { + $channel[$key] = trim($value); + } + + /* + ** Prepare the image data (if any): + */ + + foreach ($image as $key => $value) { + $image[$key] = trim($value); + } + + if ($image['LINK'] && $image['URL'] && $image['TITLE']) { + // Note, we should really use theme_image() here but that only works with local images it won't work with images fetched with a URL unless PHP version > 5 + $image = '

'; + } + else { + $image = NULL; + } + + // now that the parsing & prepping is done, start setting up the feed + $feed = array(); + $feed['URL'] = $url; + $feed['CHANNEL'] = $channel; + $feed['IMAGE'] = $image; + + /* + ** We reverse the array such that we store the first item last, + ** and the last item first. In the database, the newest item + ** should be at the top. + */ + + $items = array_reverse($items); + + foreach ($items as $item) { + $this_item = array(); + + // Prepare the item: + foreach ($item as $key => $value) { + $item[$key] = trim($value); + } + + /* + ** Resolve the item's title. If no title is found, we use + ** up to 40 characters of the description ending at a word + ** boundary but not splitting potential entities. + */ + + if ($item['TITLE']) { + $this_item['TITLE'] = filter_xss($item['TITLE']); + } + else { + $this_item['TITLE'] = filter_xss(preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item['DESCRIPTION'], 40))); + } + + /* + ** Resolve the items link. + */ + // todo this block seems weird: why is there not an else if on the second block? + if ($item['LINK']) { + $this_item['LINK'] = filter_xss($item['LINK']); + } + if ($item['GUID']) { + $this_item['GUID'] = filter_xss($item['GUID']); + } + else { + $this_item['LINK'] = $url; + } + + /** + * Atom feeds have a CONTENT and/or SUMMARY tag instead of a DESCRIPTION tag + */ + if ($item['CONTENT:ENCODED']) { + $this_item['DESCRIPTION'] = filter_xss($item['CONTENT:ENCODED']); + } + else if ($item['SUMMARY']) { + $this_item['DESCRIPTION'] = filter_xss($item['SUMMARY']); + } else { + $this_item['DESCRIPTION'] = filter_xss($item['DESCRIPTION']); + } + + /* + ** Try to resolve and parse the item's publication date. If no + ** date is found, we use the current date instead. + */ + + if ($item['PUBDATE']) $date = $item['PUBDATE']; // RSS 2.0 + else if ($item['DC:DATE']) $date = $item['DC:DATE']; // Dublin core + else if ($item['DCTERMS:ISSUED']) $date = $item['DCTERMS:ISSUED']; // Dublin core + else if ($item['DCTERMS:CREATED']) $date = $item['DCTERMS:CREATED']; // Dublin core + else if ($item['DCTERMS:MODIFIED']) $date = $item['DCTERMS:MODIFIED']; // Dublin core + else if ($item['ISSUED']) $date = $item['ISSUED']; // Atom XML + else if ($item['CREATED']) $date = $item['CREATED']; // Atom XML + else if ($item['MODIFIED']) $date = $item['MODIFIED']; // Atom XML + else if ($item['PUBLISHED']) $date = $item['PUBLISHED']; // Atom XML + else if ($item['UPDATED']) $date = $item['UPDATED']; // Atom XML + else $date = 'now'; + + $timestamp = strtotime($date); // As of PHP 5.1.0, strtotime returns FALSE on failure instead of -1. + if ($timestamp <= 0) { + $timestamp = _feed_parse_w3cdtf($date); // Returns FALSE on failure + if (!$timestamp) { + $timestamp = time(); // better than nothing + } + } + + $this_item['DATE'] = $timestamp; + + /* Same for the author */ + if ($item['AUTHOR']) $this_item['AUTHOR'] = filter_xss($item['AUTHOR']); + else if ($item['DC:CREATOR']) $this_item['AUTHOR'] = filter_xss($item['DC:CREATOR']); + + $feed['ITEMS'][] = $this_item; + } + + // TODO: Should I unset the globals here? Or does it not matter... + + return $feed; +} + + +/** + * Parse the W3C date/time format, a subset of ISO 8601. PHP date parsing + * functions do not handle this format. + * See http://www.w3.org/TR/NOTE-datetime for more information. + * Originally from MagpieRSS (http://magpierss.sourceforge.net/). + * + * @param $date_str A string with a potentially W3C DTF date. + * @return A timestamp if parsed successfully or -1 if not. + */ +function _feed_parse_w3cdtf($date_str) { + if (preg_match('/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2}))?(?:([-+])(\d{2}):?(\d{2})|(Z))?/', $date_str, $match)) { + list($year, $month, $day, $hours, $minutes, $seconds) = array($match[1], $match[2], $match[3], $match[4], $match[5], $match[6]); + // calc epoch for current date assuming GMT + $epoch = gmmktime($hours, $minutes, $seconds, $month, $day, $year); + if ($match[10] != 'Z') { // Z is zulu time, aka GMT + list($tz_mod, $tz_hour, $tz_min) = array($match[8], $match[9], $match[10]); + // zero out the variables + if (!$tz_hour) { + $tz_hour = 0; + } + if (!$tz_min) { + $tz_min = 0; + } + $offset_secs = (($tz_hour * 60) + $tz_min) * 60; + // is timezone ahead of GMT? then subtract offset + if ($tz_mod == '+') { + $offset_secs *= -1; + } + $epoch += $offset_secs; + } + return $epoch; + } + else { + return FALSE; + } +} + +/** + * Call-back function used by the XML parser. + */ +function _feed_element_start($parser, $name, $attributes) { + global $item, $element, $tag, $items, $channel; + + switch ($name) { + case 'IMAGE': + case 'TEXTINPUT': + case 'CONTENT': + case 'SUMMARY': + case 'TAGLINE': + case 'SUBTITLE': + case 'LOGO': + case 'INFO': + $element = $name; + break; + case 'ID': + if ($element != 'ITEM') { + $element = $name; + } + case 'LINK': + if ($attributes['REL'] == 'alternate') { + if ($element == 'ITEM') { + $items[$item]['LINK'] = $attributes['HREF']; + } + else { + $channel['LINK'] = $attributes['HREF']; + } + } + break; + case 'ITEM': + $element = $name; + $item += 1; + break; + case 'ENTRY': + $element = 'ITEM'; + $item += 1; + break; + } + + $tag = $name; +} + +/** + * Call-back function used by the XML parser. + */ +function _feed_element_end($parser, $name) { + global $element; + + switch ($name) { + case 'IMAGE': + case 'TEXTINPUT': + case 'ITEM': + case 'ENTRY': + case 'CONTENT': + case 'INFO': + $element = ''; + break; + case 'ID': + if ($element == 'ID') { + $element = ''; + } + } +} + +/** + * Call-back function used by the XML parser. + */ +function _feed_element_data($parser, $data) { + global $channel, $element, $items, $item, $image, $tag; + + switch ($element) { + case 'ITEM': + $items[$item][$tag] .= $data; + break; + case 'IMAGE': + case 'LOGO': + $image[$tag] .= $data; + break; + case 'LINK': + if ($data) { + $items[$item][$tag] .= $data; + } + break; + case 'CONTENT': + $items[$item]['CONTENT'] .= $data; + break; + case 'SUMMARY': + $items[$item]['SUMMARY'] .= $data; + break; + case 'TAGLINE': + case 'SUBTITLE': + $channel['DESCRIPTION'] .= $data; + break; + case 'INFO': + case 'ID': + case 'TEXTINPUT': + // The sub-element is not supported. However, we must recognize + // it or its contents will end up in the item array. + break; + default: + $channel[$tag] .= $data; + } +} === modified file 'includes/common.inc' --- includes/common.inc +++ includes/common.inc @@ -1349,6 +1349,7 @@ require_once './includes/unicode.inc'; require_once './includes/image.inc'; require_once './includes/form.inc'; + require_once './includes/feed.inc'; // Set the Drupal custom error handler. set_error_handler('error_handler'); // Emit the correct charset HTTP header. === modified file 'modules/aggregator.module' --- modules/aggregator.module +++ modules/aggregator.module @@ -572,349 +572,66 @@ drupal_set_message(t('The news items from %site have been removed.', array('%site' => theme('placeholder', $feed['title'])))); } -/** - * Call-back function used by the XML parser. - */ -function aggregator_element_start($parser, $name, $attributes) { - global $item, $element, $tag, $items, $channel; - - switch ($name) { - case 'IMAGE': - case 'TEXTINPUT': - case 'CONTENT': - case 'SUMMARY': - case 'TAGLINE': - case 'SUBTITLE': - case 'LOGO': - case 'INFO': - $element = $name; - break; - case 'ID': - if ($element != 'ITEM') { - $element = $name; - } - case 'LINK': - if ($attributes['REL'] == 'alternate') { - if ($element == 'ITEM') { - $items[$item]['LINK'] = $attributes['HREF']; - } - else { - $channel['LINK'] = $attributes['HREF']; - } - } - break; - case 'ITEM': - $element = $name; - $item += 1; - break; - case 'ENTRY': - $element = 'ITEM'; - $item += 1; - break; - } - - $tag = $name; -} - -/** - * Call-back function used by the XML parser. - */ -function aggregator_element_end($parser, $name) { - global $element; - - switch ($name) { - case 'IMAGE': - case 'TEXTINPUT': - case 'ITEM': - case 'ENTRY': - case 'CONTENT': - case 'INFO': - $element = ''; - break; - case 'ID': - if ($element == 'ID') { - $element = ''; - } - } -} - -/** - * Call-back function used by the XML parser. - */ -function aggregator_element_data($parser, $data) { - global $channel, $element, $items, $item, $image, $tag; - - switch ($element) { - case 'ITEM': - $items[$item][$tag] .= $data; - break; - case 'IMAGE': - case 'LOGO': - $image[$tag] .= $data; - break; - case 'LINK': - if ($data) { - $items[$item][$tag] .= $data; - } - break; - case 'CONTENT': - $items[$item]['CONTENT'] .= $data; - break; - case 'SUMMARY': - $items[$item]['SUMMARY'] .= $data; - break; - case 'TAGLINE': - case 'SUBTITLE': - $channel['DESCRIPTION'] .= $data; - break; - case 'INFO': - case 'ID': - case 'TEXTINPUT': - // The sub-element is not supported. However, we must recognize - // it or its contents will end up in the item array. - break; - default: - $channel[$tag] .= $data; - } -} /** * Checks a news feed for new items. */ function aggregator_refresh($feed) { - global $channel, $image; - - // Generate conditional GET headers. - $headers = array(); - if ($feed['etag']) { - $headers['If-None-Match'] = $feed['etag']; - } - if ($feed['modified']) { - $headers['If-Modified-Since'] = gmdate('D, d M Y H:i:s', $feed['modified']) .' GMT'; - } // Request feed. - $result = drupal_http_request($feed['url'], $headers); - - // Process HTTP response code. - switch ($result->code) { - case 304: - db_query('UPDATE {aggregator_feed} SET checked = %d WHERE fid = %d', time(), $feed['fid']); - drupal_set_message(t('There is no new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); - break; - case 301: - $feed['url'] = $result->redirect_url; - watchdog('aggregator', t('Updated URL for feed %title to %url.', array('%title' => theme('placeholder', $feed['title']), '%url' => theme('placeholder', $feed['url'])))); - - case 200: - case 302: - case 307: - // Filter the input data: - if (aggregator_parse_feed($result->data, $feed)) { + $request = drupal_retrieve_feed($feed['url'], $feed['modified'], $feed['etag']); - if ($result->headers['Last-Modified']) { - $modified = strtotime($result->headers['Last-Modified']); - } - - /* - ** Prepare the channel data: - */ - - foreach ($channel as $key => $value) { - $channel[$key] = trim($value); - } - - /* - ** Prepare the image data (if any): - */ - - foreach ($image as $key => $value) { - $image[$key] = trim($value); - } - - if ($image['LINK'] && $image['URL'] && $image['TITLE']) { - // Note, we should really use theme_image() here but that only works with local images it won't work with images fetched with a URL unless PHP version > 5 - $image = '

'; - } - else { - $image = NULL; - } - - /* - ** Update the feed data: - */ - - db_query("UPDATE {aggregator_feed} SET url = '%s', checked = %d, link = '%s', description = '%s', image = '%s', etag = '%s', modified = %d WHERE fid = %d", $feed['url'], time(), $channel['LINK'], $channel['DESCRIPTION'], $image, $result->headers['ETag'], $modified, $feed['fid']); - - /* - ** Clear the cache: - */ - - cache_clear_all(); - - watchdog('aggregator', t('There is new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); - drupal_set_message(t('There is new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); - } - break; - default: + // Process new feed. + if(is_null($request)) { + /* old errors, for posterity's sake watchdog('aggregator', t('The RSS-feed from %site seems to be broken, due to "%error".', array('%site' => theme('placeholder', $feed['title']), '%error' => theme('placeholder', $result->code .' '. $result->error))), WATCHDOG_WARNING); drupal_set_message(t('The RSS-feed from %site seems to be broken, because of error "%error".', array('%site' => theme('placeholder', $feed['title']), '%error' => theme('placeholder', $result->code .' '. $result->error)))); + */ } -} - -/** - * Parse the W3C date/time format, a subset of ISO 8601. PHP date parsing - * functions do not handle this format. - * See http://www.w3.org/TR/NOTE-datetime for more information. - * Originally from MagpieRSS (http://magpierss.sourceforge.net/). - * - * @param $date_str A string with a potentially W3C DTF date. - * @return A timestamp if parsed successfully or -1 if not. - */ -function aggregator_parse_w3cdtf($date_str) { - if (preg_match('/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2}))?(?:([-+])(\d{2}):?(\d{2})|(Z))?/', $date_str, $match)) { - list($year, $month, $day, $hours, $minutes, $seconds) = array($match[1], $match[2], $match[3], $match[4], $match[5], $match[6]); - // calc epoch for current date assuming GMT - $epoch = gmmktime($hours, $minutes, $seconds, $month, $day, $year); - if ($match[10] != 'Z') { // Z is zulu time, aka GMT - list($tz_mod, $tz_hour, $tz_min) = array($match[8], $match[9], $match[10]); - // zero out the variables - if (!$tz_hour) { - $tz_hour = 0; - } - if (!$tz_min) { - $tz_min = 0; - } - $offset_secs = (($tz_hour * 60) + $tz_min) * 60; - // is timezone ahead of GMT? then subtract offset - if ($tz_mod == '+') { - $offset_secs *= -1; - } - $epoch += $offset_secs; - } - return $epoch; - } + else if(empty($request)) { + db_query('UPDATE {aggregator_feed} SET checked = %d WHERE fid = %d', time(), $feed['fid']); + drupal_set_message(t('There is no new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); + } else { - return FALSE; - } -} - -function aggregator_parse_feed(&$data, $feed) { - global $items, $image, $channel; - - // Unset the global variables before we use them: - unset($GLOBALS['element'], $GLOBALS['item'], $GLOBALS['tag']); - $items = array(); - $image = array(); - $channel = array(); - - // parse the data: - $xml_parser = drupal_xml_parser_create($data); - xml_set_element_handler($xml_parser, 'aggregator_element_start', 'aggregator_element_end'); - xml_set_character_data_handler($xml_parser, 'aggregator_element_data'); - - if (!xml_parse($xml_parser, $data, 1)) { - watchdog('aggregator', t('The RSS-feed from %site seems to be broken, due to an error "%error" on line %line.', array('%site' => theme('placeholder', $feed['title']), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser))), WATCHDOG_WARNING); - drupal_set_message(t('The RSS-feed from %site seems to be broken, because of error "%error" on line %line.', array('%site' => theme('placeholder', $feed['title']), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser))), 'error'); - return 0; - } - xml_parser_free($xml_parser); - - /* - ** We reverse the array such that we store the first item last, - ** and the last item first. In the database, the newest item - ** should be at the top. - */ - - $items = array_reverse($items); - - foreach ($items as $item) { - unset($title, $link, $author, $description, $guid); - - // Prepare the item: - foreach ($item as $key => $value) { - $item[$key] = trim($value); - } - - /* - ** Resolve the item's title. If no title is found, we use - ** up to 40 characters of the description ending at a word - ** boundary but not splitting potential entities. - */ - - if ($item['TITLE']) { - $title = $item['TITLE']; - } - else { - $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item['DESCRIPTION'], 40)); - } - - /* - ** Resolve the items link. - */ - - if ($item['LINK']) { - $link = $item['LINK']; - } - if ($item['GUID']) { - $guid = $item['GUID']; - } - else { - $link = $feed['link']; - } - - /** - * Atom feeds have a CONTENT and/or SUMMARY tag instead of a DESCRIPTION tag - */ - if ($item['CONTENT:ENCODED']) { - $item['DESCRIPTION'] = $item['CONTENT:ENCODED']; - } - else if ($item['SUMMARY']) { - $item['DESCRIPTION'] = $item['SUMMARY']; - } - - /* - ** Try to resolve and parse the item's publication date. If no - ** date is found, we use the current date instead. - */ - - if ($item['PUBDATE']) $date = $item['PUBDATE']; // RSS 2.0 - else if ($item['DC:DATE']) $date = $item['DC:DATE']; // Dublin core - else if ($item['DCTERMS:ISSUED']) $date = $item['DCTERMS:ISSUED']; // Dublin core - else if ($item['DCTERMS:CREATED']) $date = $item['DCTERMS:CREATED']; // Dublin core - else if ($item['DCTERMS:MODIFIED']) $date = $item['DCTERMS:MODIFIED']; // Dublin core - else if ($item['ISSUED']) $date = $item['ISSUED']; // Atom XML - else if ($item['CREATED']) $date = $item['CREATED']; // Atom XML - else if ($item['MODIFIED']) $date = $item['MODIFIED']; // Atom XML - else if ($item['PUBLISHED']) $date = $item['PUBLISHED']; // Atom XML - else if ($item['UPDATED']) $date = $item['UPDATED']; // Atom XML - else $date = 'now'; - - $timestamp = strtotime($date); // As of PHP 5.1.0, strtotime returns FALSE on failure instead of -1. - if ($timestamp <= 0) { - $timestamp = aggregator_parse_w3cdtf($date); // Returns FALSE on failure - if (!$timestamp) { - $timestamp = time(); // better than nothing - } - } - + if($request['URL'] != $feed['url']) { + $feed['url'] = $request['URL']; + watchdog('aggregator', t('Updated URL for feed %title to %url.', array('%title' => theme('placeholder', $feed['title']), '%url' => theme('placeholder', $feed['url'])))); + } + + /* + ** Update the feed data: + */ + + db_query("UPDATE {aggregator_feed} SET url = '%s', checked = %d, link = '%s', description = '%s', image = '%s', etag = '%s', modified = %d WHERE fid = %d", $feed['url'], time(), $request['CHANNEL']['LINK'], $request['CHANNEL']['DESCRIPTION'], $request['CHANNEL']['IMAGE'], $request['HEADERS']['ETag'], $request['HEADERS']['MODIFIED'], $feed['fid']); + + // save the items to the database + foreach($request['ITEMS'] as $item) { /* ** Save this item. Try to avoid duplicate entries as much as ** possible. If we find a duplicate entry, we resolve it and ** pass along it's ID such that we can update it if needed. */ - if ($guid) { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND guid = '%s'", $feed['fid'], $guid)); - } - else if ($link && $link != $feed['link'] && $link != $feed['url']) { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND link = '%s'", $feed['fid'], $link)); - } - else { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND title = '%s'", $feed['fid'], $title)); + if ($item['GUID']) { + $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND guid = '%s'", $feed['fid'], $item['GUID'])); + } + else if ($item['LINK'] && $link != $request['CHANNEL']['LINK'] && $item['LINK'] != $feed['url']) { + $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND link = '%s'", $feed['fid'], $item['LINK'])); + } + else { + $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND title = '%s'", $feed['fid'], $item['TITLE'])); + } + + aggregator_save_item(array('iid' => $entry->iid, 'fid' => $feed['fid'], 'timestamp' => $item['DATE'], 'title' => $item['TITLE'], 'link' => $item['LINK'], 'author' => $item['AUTHOR'], 'description' => $item['DESCRIPTION'], 'guid' => $item['GUID'])); } + /* + ** Clear the cache: + */ + + cache_clear_all(); - aggregator_save_item(array('iid' => $entry->iid, 'fid' => $feed['fid'], 'timestamp' => $timestamp, 'title' => $title, 'link' => $link, 'author' => $item['AUTHOR'], 'description' => $item['DESCRIPTION'], 'guid' => $guid)); + watchdog('aggregator', t('There is new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); + drupal_set_message(t('There is new syndicated content from %site.', array('%site' => theme('placeholder', $feed['title'])))); } /* @@ -933,7 +650,6 @@ db_query('DELETE FROM {aggregator_item} WHERE fid = %d AND timestamp < %d', $feed['fid'], $age); } - return 1; } function aggregator_save_item($edit) {