diff --git a/mailcomment.info b/mailcomment.info index ccd811a..87841be 100644 --- a/mailcomment.info +++ b/mailcomment.info @@ -3,4 +3,5 @@ description = Gateway module for responding to posts via mail package = "Mailhandler" dependencies[] = notifications dependencies[] = mailhandler +files[] = tests/mailcomment.test core = 6.x diff --git a/mailcomment.install b/mailcomment.install index c9178ce..e4a9f90 100644 --- a/mailcomment.install +++ b/mailcomment.install @@ -16,9 +16,26 @@ function mailcomment_install() { variable_set('mailcomment_mailboxes', array($mailbox)); } +/** + * Set the new mailcomment_mailboxes variable with the mailcomment_mailbox variable by default. + */ function mailcomment_update_6001() { - // Set the new mailcomment_mailboxes variable with the mailcomment_mailbox variable by default. $mailbox = variable_get('mailcomment_mailbox', ''); variable_set('mailcomment_mailboxes', array($mailbox)); return array(array('success' => TRUE, 'query' => 'Set the mailcomment_mailboxes variable.')); -} \ No newline at end of file +} + +/** + * Add filtering modes: To mimic old behavior, use basic filtering when mailcomment_reply_text was set by user. + */ +function mailcomment_update_6002() { + if (variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) { + variable_set('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_BASIC); + $query = 'Set mailcomment_import_filter_mode variable to use basic filtering.'; + } + else { + variable_set('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_NONE); + $query = 'Set mailcomment_import_filter_mode variable to use no filtering.'; + } + return array(array('success' => TRUE, 'query' => $query)); +} diff --git a/mailcomment.module b/mailcomment.module index 114f96c..ebb953d 100644 --- a/mailcomment.module +++ b/mailcomment.module @@ -15,6 +15,11 @@ define('MAILCOMMENT_ERROR_EXPIRED', 3); // Error code, no valid user define('MAILCOMMENT_ERROR_USER', 4); +// Filter modes +define('MAILCOMMENT_FILTER_NONE', 0); +define('MAILCOMMENT_FILTER_BASIC', 1); +define('MAILCOMMENT_FILTER_AGGRESSIVE', 2); + /** * Implementation of hook_menu(). */ @@ -61,18 +66,38 @@ function mailcomment_admin_settings() { '#default_value' => variable_get('mailcomment_expiration', 0), '#description' => t('Time after which the signature of outgoing e-mails expires and responses won\'t be accepted anymore.'), ); - $form['mailcomment_reply_text'] = array( + + $form['filter_options'] = array( + '#title' => t('Message filtering'), + '#type' => 'fieldset', + '#collapsible' => TRUE, + ); + + $form['filter_options']['mailcomment_import_filter_mode'] = array( + '#title' => t('Filter mode'), + '#type' => 'radios', + '#default_value' => variable_get('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_NONE), + '#options' => array( + MAILCOMMENT_FILTER_NONE => t('No import filter. Leave messages as-is.'), + MAILCOMMENT_FILTER_BASIC => t('Basic import filter: Filter signatures and below custom reply separator.'), + MAILCOMMENT_FILTER_AGGRESSIVE => t('Aggressive: Filter below custom reply separator and common mail client text strings.'), + ), + '#description' => t('Should messages be filtering during import? If No import filter is selected, messages will be stored in the database as they are received.'), + ); + + $form['filter_options']['mailcomment_reply_text'] = array( '#title' => t('Reply text'), '#type' => 'textfield', '#default_value' => variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))')), - '#description' => t('Text to separate reply from the rest of the e-mail. Leave blank for not using this feature.'), + '#description' => t('Text to separate reply from the rest of the e-mail. Leave blank to disable.'), ); - $form['mailcomment_insert_reply_text'] = array( + $form['filter_options']['mailcomment_insert_reply_text'] = array( '#title' => t('Insert reply to text'), '#type' => 'checkbox', '#default_value' => variable_get('mailcomment_insert_reply_text', 1), '#description' => t('If this box is checked, mail comment will insert the text specified as the Reply text at the top of messages sent. You may uncheck this box and insert the text manually into a messaging template (for example, above the footer).'), ); + $form['mailcomment_error_bounce'] = array( '#title' => t('Bounce rejected emails'), '#type' => 'checkbox', @@ -509,17 +534,23 @@ function mailcomment_signature($params) { return md5(implode('-', $params)); } -// Need to clean up body text after the comment is parsed and before it is processed +/** + * Implementation of hook_feeds_after_parse(). + * + * Apply filters before writing to database. + */ function mailcomment_feeds_after_parse(FeedsImporter $importer, FeedsSource $source) { - // Now trim out the rest of the message if separator text exists - // @ TODO May fail for html mails - if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) { - foreach ($source->batch->items as &$message) { - $pos = strpos($message['origbody'], $marker); - if ($pos !== FALSE) { - $message['origbody'] = drupal_substr($message['origbody'], 0, $pos); + switch (variable_get('mailcomment_import_filter_mode', 0)) { + case MAILCOMMENT_FILTER_BASIC: + foreach ($source->batch->items as &$message) { + $message['origbody'] = _mailcomment_filter_basic($message['origbody']); } - } + break; + case MAILCOMMENT_FILTER_AGGRESSIVE: + foreach ($source->batch->items as &$message) { + $message['origbody'] = _mailcomment_filter_aggressive($message['origbody']); + } + break; } } @@ -561,51 +592,76 @@ function mailcomment_filter_description($delta = 0) { function mailcomment_filter_prepare($delta = 0, $format = -1, $text = '', $cache_id = 0) { switch ($delta) { case '0': - // Get the offset of the last two line breaks and substring. - $search = preg_match_all("/|\r\n\r\n|\n\n[^|\r\n\r\n|\n\n]+$/m", $text, $matches, PREG_OFFSET_CAPTURE); - if (!empty($matches[0])) { - $key = max(array_keys($matches[0])); - $offset = $matches[0][$key][1]; - $text = drupal_substr($text, 0, $offset); - } - return $text; + return _mailcomment_filter_basic($text); case '1': - /* - * Below are what each pattern tries to match. - * gmail 1 => On 3/10/09, - * gmail 2 => On Mon, Mar 9, 2009 at 10:14 PM, - * mac_mail_1 => On Feb 17, 2009, at 2:30 PM, foo@example.com wrote: - * mac_mail_2 => On Feb 17, 2009, at 2:30 PM, - * outlook_express => ----- Original Message ----- - * outlook_legacy => From: foo@example.com [foo@example.com] - * - */ - // Build an array of different possible matching expressions - $expressions = array( - 'gmail_1' => "/On \d{1,2}[\/\-]{1}\d{1,2}[\/\-]{1}\d{1,2},/", - 'gmail_2' => "/On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M/", - 'mac_mail_1' => "/On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M, [a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4} wrote/", - 'mac_mail_2' => "/On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M,/", // currently mac mail uses a name rather than the email address - 'outlook_express' => '/-----\s?Original Message\s?-----/', - 'outlook_legacy' => '/From:(\s*)[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}(\s*)\[mailto:[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}\]/', - 'generic_1' => '/From:.*Sent:.*To:.*Subject:/msU', - ); - - $match = 0; - foreach ($expressions as $name => $expression) { - if ($match < 1) { - $split = preg_split($expression, $text); - //$split = preg_split("/\>/",$split[0]); - if (count($split) > 1) { - // An expression was matched, so break the loop. - $match = 1; - } - } - } + return _mailcomment_filter_aggressive($text); + } +} - if (!empty($split[1])) { - $text = $split[0]; - } - return $text; +/** + * Filter a message for signature in a simple way. + * + * @param $text + * Text to filter. + * @return + * A filtered string. + */ +function _mailcomment_filter_basic($text) { + // Get the offset of the last two line breaks and substring. + $search = preg_match_all("/|\r\n\r\n|\n\n[^|\r\n\r\n|\n\n]+$/m", $text, $matches, PREG_OFFSET_CAPTURE); + if (!empty($matches[0])) { + $key = max(array_keys($matches[0])); + $offset = $matches[0][$key][1]; + $text = trim(drupal_substr($text, 0, $offset)); + } + + if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) { + $search = preg_match_all("/[\> ]*". preg_quote($marker) ."/", $text, $matches, PREG_OFFSET_CAPTURE); + if (!empty($matches[0])) { + $key = max(array_keys($matches[0])); + $offset = $matches[0][$key][1]; + $text = trim(drupal_substr($text, 0, $offset)); + } + } + + return $text; +} + +/** + * Aggressively filter a message for quotes and signatures. + * + * @param $text + * Text to filter. + * @return + * A filtered string. + */ +function _mailcomment_filter_aggressive($text) { + $delimiters = array(); + + $delimiters = array( + "On \d{1,2}[\/\-]{1}\d{1,2}[\/\-]{1}\d{1,2},", // Gmail form 1 + "On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M", // Gmail form 1 + "On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M", // Gmail form 2 + "On \d{1,2}\/\d{1,2}\/\d{4} \d{1,2}:\d{1,2} [AP]M,", // Thunderbird + "On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M, [a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4} wrote", // Mac mail + "On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M,", // Mac mail + '-----\s?Original Message\s?-----', // Outlook express + 'From:(\s*)[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}(\s*)\[mailto:[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}\]', // Outlook legacy + '[_]*\nFrom:', // Yahoo + Outlook web + 'From:.*Sent:.*To:.*Subject:', // Fallback + ); + + if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) { + $delimiters[] = '[\> ]*'. preg_quote($marker); + } + + $expression = '/('. implode('|', $delimiters) .')/'; + + if (preg_match($expression, $text, $matches, PREG_OFFSET_CAPTURE)) { + $text = trim(drupal_substr($text, 0, $matches[0][1])); } + + return $text; } + + diff --git a/tests/mailcomment.test b/tests/mailcomment.test new file mode 100644 index 0000000..b0cd306 --- /dev/null +++ b/tests/mailcomment.test @@ -0,0 +1,119 @@ + 'Mailcomment', + 'description' => 'Test mailcomment message parsing.', + 'group' => 'mailcomment', + ); + } + + /** + * setUp() performs any pre-requisite tasks that need to happen. + */ + public function setUp() { + variable_set('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))')); + + // Enable any modules required for the test. + parent::setUp( + 'ctools', + 'notifications', + 'messaging', + 'feeds', + 'autoload', + 'token', + 'job_scheduler', + 'messaging_template', + 'mailhandler', + 'mailcomment' + ); + } + + public static function mailCommentTestMessages() { + $messages = array(); + + // Gmail + $messages['gmail'] = "Hello world. + +On Fri, Aug 19, 2011 at 9:51 AM, test person wrote: +> ((( Reply ABOVE this LINE to POST a COMMENT ))) +> +> user said: +> +> Courgette lettuce bell pepper asparagus collard greens epazote broccoli rabe. Wattle +> seed cress welsh onion broccoli rabe pea sprouts tomatillo celery ricebean napa cabbage +> bush tomato daikon horseradish turnip winter purslane. Chickweed radicchio taro tigernut +> spinach tomatillo black-eyed pea avocado. + +-- +Test person name | http://hostname.tld"; + + // Thunderbird + $messages['thunderbird'] = "Hello world. + +On 08/18/2011 04:19 PM, Test Person wrote: +> ((( Reply ABOVE this LINE to POST a COMMENT ))) +> +> user said: +> +> Mustard scallion yarrow aubergine sierra leone bologi sea lettuce +> chicory broccoli collard greens turnip greens shallot radicchio komatsuna. +> Broccoli celery grape bell pepper kohlrabi radicchio sierra leone bologi."; + + // Yahoo mail + $messages['yahoo'] = "Hello world. +________________________________ +From: Firstname Lastname +To: recipient@hostname.tld +Sent: Tuesday, September 13, 2011 10:14 AM +Subject: Re: Test subject + +((( Reply ABOVE this LINE to POST a COMMENT ))) + +Earthnut pea eggplant brussels sprout bitterleaf quandong desert raisin wattle +seed celtuce bamboo shoot mung bean cauliflower maize salad chickpea celery. +Winter purslane silver beet daikon brussels sprout broccoli rabe lotus root +water chestnut garlic melon lettuce potato pumpkin. Ricebean groundnut potato +celery corn artichoke sea lettuce. No joke."; + + // Outlook web + $messages['outlook_web'] = "Hello world. +________________________________________ +From: user@hostname.tld [user@hostname.tld] +Sent: Thursday, August 18, 2011 4:19 PM +To: Recipient Name +Subject: Re: [Site-name] Subject + +((( Reply ABOVE this LINE to POST a COMMENT ))) + +Test user said: + +Pea cress water spinach summer purslane asparagus scallion squash sorrel. Pea +green bean celery chickpea lotus root parsnip celery arugula carrot winter +purslane cress plantain tatsoi pumpkin pea ricebean mung bean lentil."; + + $messages['generic_reply'] = "Hello world. + +> ((( Reply ABOVE this LINE to POST a COMMENT ))) +> +> user said: +> +> Mustard scallion yarrow aubergine sierra leone bologi sea lettuce +> chicory broccoli collard greens turnip greens shallot radicchio komatsuna. +> Broccoli celery grape bell pepper kohlrabi radicchio sierra leone bologi."; + + return $messages; + } + + public function testMailCommentParseMessage() { + $messages = $this->mailCommentTestMessages(); + foreach ($messages as $key => $message) { + $text = _mailcomment_filter_aggressive($message); + $this->assertEqual($text, 'Hello world.', 'Filtered '. $key); + } + } +}