Index: modules/filter/filter.module =================================================================== RCS file: /cvs/drupal/drupal/modules/filter/filter.module,v retrieving revision 1.338 diff -u -p -r1.338 filter.module --- modules/filter/filter.module 17 Aug 2010 13:50:52 -0000 1.338 +++ modules/filter/filter.module 18 Aug 2010 22:19:54 -0000 @@ -1293,46 +1293,150 @@ function _filter_url_settings($form, &$f } /** - * URL filter. Automatically converts text web addresses (URLs, e-mail addresses, - * ftp links, etc.) into hyperlinks. + * URL filter. Automatically converts text into hyperlinks. + * + * This filter identifies and makes clickable three types of "links". + * - URLs like http://example.com. + * - E-mail addresses like name@example.com. + * - Web addresses without the "http://" protocol defined, like www.example.com. + * Each type must be processed separately, as there is no one regular + * expression that could possibly match all of the cases in one pass. */ function _filter_url($text, $filter) { - // Pass length to regexp callback + // Tags to skip and not recurse into. + $ignore_tags = 'a|script|style|code|pre'; + + // Pass length to regexp callback. _filter_url_trim(NULL, $filter->settings['filter_url_length']); - $text = ' ' . $text . ' '; + // Create an array which contains the regexps for each type of link. + // The key to the regexp is the name of a function that is used as + // callback function to process matches of the regexp. The callback function + // is to return the replacement for the match. The array is used and + // matching/replacement done below inside some loops. + $tasks = array(); + + // Prepare protocols pattern for absolute URLs. + // check_url() will replace any bad protocols with HTTP, so we need to support + // the identical list. While '//' is technically optional for MAILTO only, + // we cannot cleanly differ between protocols here without hard-coding MAILTO, + // so '//' is optional for all protocols. + // @see filter_xss_bad_protocol() + $protocols = variable_get('filter_allowed_protocols', array('http', 'https', 'ftp', 'news', 'nntp', 'telnet', 'mailto', 'irc', 'ssh', 'sftp', 'webcal', 'rtsp')); + $protocols = implode(':(?://)?|', $protocols) . ':(?://)?'; + + // Prepare domain name pattern. + // The ICANN seems to be on track towards accepting more diverse top level + // domains, so this pattern has been "future-proofed" to allow for TLDs + // of length 2-64. + $domain = '(?:[A-Za-z0-9._+-]+\.)?[A-Za-z]{2,64}\b'; + $ip = '(?:[0-9]{1,3}\.){3}[0-9]{1,3}'; + $auth = '[a-zA-Z0-9:%_+*~#?&=.,/;-]+@'; + $trail = '[a-zA-Z0-9:%_+*~#&\[\]=/;?.,-]*[a-zA-Z0-9:%_+*~#&\[\]=/;-]'; // Match absolute URLs. - $text = preg_replace_callback("`(

|

  • ||[ \n\r\t\(])((http://|https://|ftp://|mailto:|smb://|afp://|file://|gopher://|news://|ssl://|sslv2://|sslv3://|tls://|tcp://|udp://)([a-zA-Z0-9@:%_+*~#?&=.,/;-]*[a-zA-Z0-9@:%_+*~#&=/;-]))([.,?!]*?)(?=(

    |
    • ||[ \n\r\t\)]))`i", '_filter_url_parse_full_links', $text); + $url_pattern = "(?:$auth)?(?:$domain|$ip)/?(?:$trail)?"; + $pattern = "`((?:$protocols)(?:$url_pattern))([\.,?!]?)`i"; + $tasks['_filter_url_parse_full_links'] = $pattern; // Match e-mail addresses. - $text = preg_replace("`(

    |

  • ||[ \n\r\t\(])([A-Za-z0-9._-]+@[A-Za-z0-9._+-]+\.[A-Za-z]{2,4})([.,?!]*?)(?=(

    |
    • ||[ \n\r\t\)]))`i", '\1\2\3', $text); + $url_pattern = "[A-Za-z0-9._-]+@(?:$domain)"; + $pattern = "`($url_pattern)`i"; + $tasks['_filter_url_parse_email_links'] = $pattern; + + // Match www domains. + $url_pattern = "www\.(?:$domain)/?(?:$trail)?"; + $pattern = "`($url_pattern)([\.,?!]?)`i"; + $tasks['_filter_url_parse_partial_links'] = $pattern; + + // Each type of URL needs to be processed separately. The text is joined and + // re-split after each task, since all injected HTML tags must be correctly + // protected before the next task. + foreach ($tasks as $task => $pattern) { + // Split at all tags; ensures that no tags or attributes are processed. + $chunks = preg_split('/(<.+?>)/i', $text, -1, PREG_SPLIT_DELIM_CAPTURE); + // PHP ensures that the array consists of alternating delimiters and + // literals, and begins and ends with a literal (inserting NULL as + // required). Therefore, the first chunk is always text: + $chunk_type = 'text'; + // If a tag of $ignore_tags is found, it is stored in $open_tag and only + // removed when the closing tag is found. Until the closing tag is found, + // no replacements are made. + $open_tag = ''; + + for ($i = 0; $i < count($chunks); $i++) { + if ($chunk_type == 'text') { + // Only process this text if there are no unclosed $ignore_tags. + if ($open_tag == '') { + // If there is a match, inject a link into this chunk via the callback + // function contained in $task. + $chunks[$i] = preg_replace_callback($pattern, $task, $chunks[$i]); + } + // Text chunk is done, so next chunk must be a tag. + $chunk_type = 'tag'; + } + else { + // Only process this tag if there are no unclosed $ignore_tags. + if ($open_tag == '') { + // Check whether this tag is contained in $ignore_tags. + if (preg_match("`<($ignore_tags)(?:\s|>)`i", $chunks[$i], $matches)) { + $open_tag = $matches[1]; + } + } + // Otherwise, check whether this is the closing tag for $open_tag. + else { + if (preg_match("`<\/$open_tag>`i", $chunks[$i], $matches)) { + $open_tag = ''; + } + } + // Tag chunk is done, so next chunk must be text. + $chunk_type = 'text'; + } + } - // Match www domains/addresses. - $text = preg_replace_callback("`(

    |

  • |[ \n\r\t\(])(www\.[a-zA-Z0-9@:%_+*~#?&=.,/;-]*[a-zA-Z0-9@:%_+~#\&=/;-])([.,?!]*?)(?=(

    |
    • ||[ \n\r\t\)]))`i", '_filter_url_parse_partial_links', $text); - $text = substr($text, 1, -1); + $text = implode($chunks); + } return $text; } /** - * Make links out of absolute URLs. + * preg_replace callback to make links out of absolute URLs. */ function _filter_url_parse_full_links($match) { - $match[2] = decode_entities($match[2]); - $caption = check_plain(_filter_url_trim($match[2])); - $match[2] = check_url($match[2]); - return $match[1] . '' . $caption . '' . $match[5]; + // The $i:th parenthesis in the regexp contains the URL. + $i = 1; + + $match[$i] = decode_entities($match[$i]); + $caption = check_plain(_filter_url_trim($match[$i])); + $match[$i] = check_url($match[$i]); + return '' . $caption . '' . $match[$i + 1]; } /** - * Make links out of domain names starting with "www." + * preg_replace callback to make links out of e-mail addresses. + */ +function _filter_url_parse_email_links($match) { + // The $i:th parenthesis in the regexp contains the URL. + $i = 0; + + $match[$i] = decode_entities($match[$i]); + $caption = check_plain(_filter_url_trim($match[$i])); + $match[$i] = check_plain($match[$i]); + return '' . $caption . ''; +} + +/** + * preg_replace callback to make links out of domain names starting with "www." */ function _filter_url_parse_partial_links($match) { - $match[2] = decode_entities($match[2]); - $caption = check_plain(_filter_url_trim($match[2])); - $match[2] = check_plain($match[2]); - return $match[1] . '' . $caption . '' . $match[3]; + // The $i:th parenthesis in the regexp contains the URL. + $i = 1; + + $match[$i] = decode_entities($match[$i]); + $caption = check_plain(_filter_url_trim($match[$i])); + $match[$i] = check_plain($match[$i]); + return '' . $caption . '' . $match[$i + 1]; } /** @@ -1345,8 +1449,8 @@ function _filter_url_trim($text, $length } // Use +3 for '...' string length. - if (strlen($text) > $_length + 3) { - $text = substr($text, 0, $_length) . '...'; + if (drupal_strlen($text) > $_length + 3) { + $text = drupal_substr($text, 0, $_length) . '...'; } return $text; Index: modules/filter/filter.test =================================================================== RCS file: /cvs/drupal/drupal/modules/filter/filter.test,v retrieving revision 1.71 diff -u -p -r1.71 filter.test --- modules/filter/filter.test 5 Aug 2010 23:53:38 -0000 1.71 +++ modules/filter/filter.test 18 Aug 2010 22:23:14 -0000 @@ -981,84 +981,256 @@ class FilterUnitTestCase extends DrupalU } /** - * Test the URL filter. + * Tests the URL filter. */ function testUrlFilter() { // Setup dummy filter object. - $filter = new stdClass(); + $filter = new stdClass; $filter->settings = array( 'filter_url_length' => 496, ); + // @todo Possible categories: + // - absolute, mail, partial + // - characters/encoding, surrounding markup, security + + // Filter selection/pattern matching. + $tests = array( + // HTTP URLs. + 'http://example.com or www.example.com' => array( + 'http://example.com' => TRUE, + 'example.com' => FALSE, + 'www.example.com' => TRUE, + 'http://www.example.com' => FALSE, + ), + // MAILTO URLs. + 'person@example.com or mailto:person2@example.com' => array( + 'person@example.com' => TRUE, + 'person@example.com' => FALSE, + 'person@example.com' => FALSE, + 'mailto:person2@example.com' => TRUE, + ), + // URI parts. + 'http://trailingslash.com/ or www.trailingslash.com/ +http://host.com/some/path?query=foo&bar[baz]=beer#fragment or www.host.com/some/path?query=foo&bar[baz]=beer#fragment +ftp://user:pass@ftp.example.com/~home/dir1 +sftp://user@nonstandardport:222/dir +ssh://192.168.0.100/srv/git/drupal.git +' => array( + 'http://trailingslash.com/' => TRUE, + 'www.trailingslash.com/' => TRUE, + 'http://host.com/some/path?query=foo&bar[baz]=beer#fragment' => TRUE, + 'www.host.com/some/path?query=foo&bar[baz]=beer#fragment' => TRUE, + 'ftp://user:pass@ftp.example.com/~home/dir1' => TRUE, + 'sftp://user@nonstandardport:222/dir' => TRUE, + 'ssh://192.168.0.100/srv/git/drupal.git' => TRUE, + ), + // Encoding. + 'http://ampersand.com/?a=1&b=2 +http://encoded.com/?a=1&b=2' => array( + 'http://ampersand.com/?a=1&b=2' => TRUE, + 'http://encoded.com/?a=1&b=2' => TRUE, + ), + // Domain name length. + 'www.ex.ex or www.example.example or www.toolongdomainexampledomainexampledomainexampledomainexampledomain or +me@me.tv' => array( + 'www.ex.ex' => TRUE, + 'www.example.example' => TRUE, + 'http://www.toolong' => FALSE, + 'me@me.tv' => TRUE, + ), + // Absolute URL protocols. + 'https://example.com, +ftp://ftp.example.com, +news://example.net, +telnet://example, +irc://example.host, +ssh://odd.geek, +sftp://secure.host?, +webcal://calendar, +rtsp://127.0.0.1, +not foo://disallowed.com.' => array( + 'href="https://example.com"' => TRUE, + 'href="ftp://ftp.example.com"' => TRUE, + 'href="news://example.net"' => TRUE, + 'href="telnet://example"' => TRUE, + 'href="irc://example.host"' => TRUE, + 'href="ssh://odd.geek"' => TRUE, + 'href="sftp://secure.host"' => TRUE, + 'href="webcal://calendar"' => TRUE, + 'href="rtsp://127.0.0.1"' => TRUE, + 'href="foo://disallowed.com"' => FALSE, + 'not foo://disallowed.com.' => TRUE, + ), + ); + $this->assertFilteredString($filter, $tests); - // Converting URLs. - $f = _filter_url('http://www.example.com/', $filter); - $this->assertEqual($f, 'http://www.example.com/', t('Converting URLs.')); - - $f = _filter_url('http://www.example.com/?a=1&b=2', $filter); - $this->assertEqual($f, 'http://www.example.com/?a=1&b=2', t('Converting URLs -- ampersands.')); - - $f = _filter_url('ftp://user:pass@ftp.example.com/dir1/dir2', $filter); - $this->assertEqual($f, 'ftp://user:pass@ftp.example.com/dir1/dir2', t('Converting URLs -- FTP scheme.')); - - // Converting domain names. - $f = _filter_url('www.example.com', $filter); - $this->assertEqual($f, 'www.example.com', t('Converting domain names.')); - - $f = _filter_url('
  • www.example.com
    • ', $filter); - $this->assertEqual($f, '
  • www.example.com
    • ', t('Converting domain names -- domain in a list.')); - - $f = _filter_url('(www.example.com/dir?a=1&b=2#a)', $filter); - $this->assertEqual($f, '(www.example.com/dir?a=1&b=2#a)', t('Converting domain names -- domain in parentheses.')); - - // Converting e-mail addresses. - $f = _filter_url('johndoe@example.com', $filter); - $this->assertEqual($f, 'johndoe@example.com', t('Converting e-mail addresses.')); + // Surrounding text/punctuation. + $tests = array( + // Even though a trailing period can indicate a fully qualified domain + // name, such usage is rare compared to using a URL at the end of a + // sentence, so trailing periods should be excluded. + 'Partial URL with trailing period www.partial.com. +E-mail with trailing period person@example.com. +Absolute URL with trailing period http://www.absolute.com. +Query string with trailing period www.query.com/index.php?a=. +' => array( + 'www.partial.com.' => TRUE, + 'person@example.com.' => TRUE, + 'http://www.absolute.com.' => TRUE, + 'www.query.com/index.php?a=.' => TRUE, + ), + '(www.parenthesis.com/dir?a=1&b=2#a) +' => array( + '(www.parenthesis.com/dir?a=1&b=2#a)' => TRUE, + ), + ); + $this->assertFilteredString($filter, $tests); - $f = _filter_url('aaa@sub.tv', $filter); - $this->assertEqual($f, 'aaa@sub.tv', t('Converting e-mail addresses -- a short e-mail from Tuvalu.')); + // Surrounding markup. + $tests = array( + '

    +

    +An anchor. +

    ' => array( + '

    ' => TRUE, + '

    ' => TRUE, + 'href="http://www.namespace.com"' => FALSE, + 'href="http://namespace.com"' => FALSE, + 'An anchor.' => TRUE, + ), + 'Not www.relative.com or www.absolute.com +but http://www.strong.net or www.emphasis.info' => array( + 'www.relative.com' => TRUE, + 'href="http://www.relative.com"' => FALSE, + 'www.absolute.com' => TRUE, + 'http://www.strong.net' => TRUE, + 'www.emphasis.info' => TRUE, + ), + 'Test using www.example.com the code tag.' => array( + 'href' => FALSE, + 'http' => FALSE, + ), + 'Intro. +

    +Quoted text linking to www.example.com, written by person@example.com, originating from http://origin.example.com. @see www.usage.example.com or www.example.info bla bla. +
    + +Outro.' => array( + 'href="http://www.example.com"' => TRUE, + 'href="mailto:person@example.com"' => TRUE, + 'href="http://origin.example.com"' => TRUE, + 'http://www.usage.example.com' => FALSE, + 'http://www.example.info' => FALSE, + 'Intro.' => TRUE, + 'Outro.' => TRUE, + ), + 'Unknown tag containing x and www.example.com? And a tag beginning with p and containing www.example.pooh with p?' => array( + 'href="http://www.example.com"' => TRUE, + 'href="http://www.example.pooh"' => TRUE, + ), + '

    Test <br/>: This is a www.example17.com example with various http://www.example18.com tags. *
    + It is important www.example19.com to *
    test different URLs and http://www.example20.com in the same paragraph. *
    +HTML www.example21.com soup by person@example22.com can litererally http://www.example23.com contain *img* anything. Just a www.example24.com with http://www.example25.com thrown in. www.example26.com from person@example27.com with extra http://www.example28.com.' => array( + 'href="http://www.example17.com"' => TRUE, + 'href="http://www.example18.com"' => TRUE, + 'href="http://www.example19.com"' => TRUE, + 'href="http://www.example20.com"' => TRUE, + 'href="http://www.example21.com"' => TRUE, + 'href="mailto:person@example22.com"' => TRUE, + 'href="http://www.example23.com"' => TRUE, + 'href="http://www.example24.com"' => TRUE, + 'href="http://www.example25.com"' => TRUE, + 'href="http://www.example26.com"' => TRUE, + 'href="mailto:person@example27.com"' => TRUE, + 'href="http://www.example28.com"' => TRUE, + ), + ' +' => array( + 'href="http://www.example.com"' => FALSE, + 'href="http://example.net"' => FALSE, + ), + '' => array( + 'href' => FALSE, + ), + '' => array( + 'href' => FALSE, + ), + '

    +
    www.example.com
    +
    http://example.com
    +
    person@example.com
    +
    Check www.example.net
    +
    Some text around http://www.example.info by person@example.info?
    +
    ' => array( + 'href="http://www.example.com"' => TRUE, + 'href="http://example.com"' => TRUE, + 'href="mailto:person@example.com"' => TRUE, + 'href="http://www.example.net"' => TRUE, + 'href="http://www.example.info"' => TRUE, + 'href="mailto:person@example.info"' => TRUE, + ), + ); + $this->assertFilteredString($filter, $tests); // URL trimming. - $filter->settings['filter_url_length'] = 28; - - $f = _filter_url('http://www.example.com/d/ff.ext?a=1&b=2#a1', $filter); - $this->assertNormalized($f, 'http://www.example.com/d/ff....', t('URL trimming.')); - - // Not breaking existing links. - $f = _filter_url('www.example.com', $filter); - $this->assertEqual($f, 'www.example.com', t('Converting URLs -- do not break existing links.')); - - $f = _filter_url('http://www.example.com', $filter); - $this->assertEqual($f, 'http://www.example.com', t('Converting URLs -- do not break existing, relative links.')); - - // Addresses within some tags such as code or script should not be converted. - $f = _filter_url('http://www.example.com', $filter); - $this->assertEqual($f, 'http://www.example.com', t('Converting URLs -- skip code contents.')); - - $f = _filter_url('http://www.example.com', $filter); - $this->assertEqual($f, 'http://www.example.com', t('Converting URLs -- really skip code contents.')); - - $f = _filter_url('', $filter); - $this->assertEqual($f, '', t('Converting URLs -- do not process scripts.')); - - // Addresses in attributes should not be converted. - $f = _filter_url('

    ', $filter); - $this->assertEqual($f, '

    ', t('Converting URLs -- do not convert addresses in attributes.')); - - $f = _filter_url('text', $filter); - $this->assertEqual($f, 'text', t('Converting URLs -- do not break existing links with custom title attribute.')); - - // Even though a dot at the end of a URL can indicate a fully qualified - // domain name, such usage is rare compared to using a link at the end - // of a sentence, so remove the dot from the link. - // @todo It can also be used at the end of a filename or a query string. - $f = _filter_url('www.example.com.', $filter); - $this->assertEqual($f, 'www.example.com.', t('Converting URLs -- do not recognize a dot at the end of a domain name (FQDNs).')); - - $f = _filter_url('http://www.example.com.', $filter); - $this->assertEqual($f, 'http://www.example.com.', t('Converting URLs -- do not recognize a dot at the end of an URL (FQDNs).')); + $filter->settings['filter_url_length'] = 20; + $tests = array( + 'www.trimmed.com/d/ff.ext?a=1&b=2#a1' => array( + 'www.trimmed.com/d/ff...' => TRUE, + ), + ); + $this->assertFilteredString($filter, $tests); + } - $f = _filter_url('www.example.com/index.php?a=.', $filter); - $this->assertEqual($f, 'www.example.com/index.php?a=.', t('Converting URLs -- do forget about a dot at the end of a query string.')); + /** + * Asserts multiple filter output expectations for multiple input strings. + * + * @param $filter + * A input filter object. + * @param $tests + * An associative array, whereas each key is an arbitrary input string and + * each value is again an associative array whose keys are filter output + * strings and whose values are Booleans indicating whether the output is + * expected or not. + * + * For example: + * @code + * $tests = array( + * 'Input string' => array( + * '

    Input string

    ' => TRUE, + * 'Input string FALSE, + * ), + * ); + * @endcode + */ + protected function assertFilteredString($filter, $tests) { + foreach ($tests as $phrase => $tasks) { + $string = _filter_url($phrase, $filter); + foreach ($tasks as $value => $expected) { + // Not using assertIdentical, since combination with strpos() is hard to grok. + if ($expected) { + $this->assertTrue(strpos($string, $value) !== FALSE, t('@string: @value found.', array( + '@string' => var_export($phrase, TRUE), + '@value' => var_export($value, TRUE), + ))); + } + else { + $this->assertTrue(strpos($string, $value) === FALSE, t('@string: @value not found.', array( + '@string' => var_export($phrase, TRUE), + '@value' => var_export($value, TRUE), + ))); + } + } + } } /**