From 6843e2f742d9908718c60e2fc3712c730b821c86 Mon Sep 17 00:00:00 2001 From: Naman Gupta <01namangupta@gmail.com> Date: Wed, 4 Jul 2018 01:54:50 +0530 Subject: [PATCH] incoming email parsing feature for gmail and yahoo (#2933) * Added reply to content field in comment table * Added incoming email parsing feature for gmail and yahooo * Added incoming email parsing feature for gmail and yahooo * Minor changes * Removed column * Added filter in comment * Added mail filter in mails * Minor changes * Fixes codeclimate issues * Modified COMMENT_FILTER * Added test for parsing gmail and yahoo * Added detail readme for yahoo and gmail parsing --- app/helpers/application_helper.rb | 15 +++ app/mailers/comment_mailer.rb | 1 - app/models/comment.rb | 83 ++++++++++++-- app/models/concerns/comments_shared.rb | 5 + app/views/notes/_comment.html.erb | 10 +- test/fixtures/drupal_users.yml | 14 +++ test/fixtures/users.yml | 26 +++++ .../gmail/final_parsed_comment.txt | 11 ++ .../gmail/incoming_gmail_email.eml | 107 ++++++++++++++++++ .../gmail/incoming_gmail_email.html | 15 +++ test/incoming_test_emails/gmail/readme.md | 9 ++ .../yahoo/final_parsed_comment.txt | 10 ++ .../yahoo/incoming_yahoo_email.eml | 79 +++++++++++++ .../yahoo/incoming_yahoo_mail.html | 22 ++++ test/incoming_test_emails/yahoo/readme.md | 9 ++ test/unit/comment_test.rb | 25 +++- 16 files changed, 426 insertions(+), 15 deletions(-) create mode 100644 test/incoming_test_emails/gmail/final_parsed_comment.txt create mode 100644 test/incoming_test_emails/gmail/incoming_gmail_email.eml create mode 100644 test/incoming_test_emails/gmail/incoming_gmail_email.html create mode 100644 test/incoming_test_emails/gmail/readme.md create mode 100644 test/incoming_test_emails/yahoo/final_parsed_comment.txt create mode 100644 test/incoming_test_emails/yahoo/incoming_yahoo_email.eml create mode 100644 test/incoming_test_emails/yahoo/incoming_yahoo_mail.html create mode 100644 test/incoming_test_emails/yahoo/readme.md diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 8d0a6d816f4..ce044164183 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -114,4 +114,19 @@ def title_suggestion(comment) output end end + + def filtered_comment_body(comment_body) + if contain_trimmed_body?(comment_body) + return comment_body.split(Comment::COMMENT_FILTER).first + end + comment_body + end + + def contain_trimmed_body?(comment_body) + comment_body.include?(Comment::COMMENT_FILTER) + end + + def trimmed_body(comment_body) + comment_body.split(Comment::COMMENT_FILTER).second + end end diff --git a/app/mailers/comment_mailer.rb b/app/mailers/comment_mailer.rb index dacfb55fcc7..add90ab5d56 100644 --- a/app/mailers/comment_mailer.rb +++ b/app/mailers/comment_mailer.rb @@ -1,6 +1,5 @@ class CommentMailer < ActionMailer::Base helper :application - require 'byebug' include ApplicationHelper default from: "notifications@#{ActionMailer::Base.default_url_options[:host]}" diff --git a/app/models/comment.rb b/app/models/comment.rb index 565305a6eb1..9ab34c6f47c 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -13,6 +13,8 @@ class Comment < ApplicationRecord self.table_name = 'comments' self.primary_key = 'cid' + COMMENT_FILTER = "".freeze + def self.inheritance_column 'rails_type' end @@ -220,22 +222,79 @@ def user_reactions_map user_like_map end - def self.receive_mail(message) - node_id = message.subject[/#([\d]+)/, 1] #This took out the node ID from the subject line - puts node_id - unless node_id.nil? - node = Node.where(nid: node_id) - if node.any? - node = node.first - user = User.find_by(email: message.from.first) - if user.present? && node_id.present? - message_markdown = ReverseMarkdown.convert message.html_part.body.decoded - message_id = message.message_id - comment = node.add_comment(uid: user.uid, body: message_markdown, comment_via: 1, message_id: message_id) + def self.receive_mail(mail) + user = User.where(email: mail.from.first).first + if user + node_id = mail.subject[/#([\d]+)/, 1] #This took out the node ID from the subject line + unless node_id.nil? + node = Node.where(nid: node_id).first + if node + mail_doc = Nokogiri::HTML(mail.html_part.body.decoded) # To parse the mail to extract comment content and reply content + domain = get_domain mail.from.first + if domain == "gmail" + content = gmail_parsed_mail mail_doc + elsif domain == "yahoo" + content = yahoo_parsed_mail mail_doc + else + content = { + "comment_content" => mail_doc, + "extra_content" => nil + } + end + + if content["extra_content"].nil? + comment_content_markdown = ReverseMarkdown.convert content["comment_content"] + else + extra_content_markdown = ReverseMarkdown.convert content["extra_content"] + comment_content_markdown = ReverseMarkdown.convert content["comment_content"] + comment_content_markdown = comment_content_markdown + COMMENT_FILTER + extra_content_markdown + end + message_id = mail.message_id + comment = node.add_comment(uid: user.uid, body: comment_content_markdown, comment_via: 1, message_id: message_id) comment.notify user end end end end + def self.get_domain(email) + domain = email[/(?<=@)[^.]+(?=\.)/, 0] + end + + def self.yahoo_parsed_mail(mail_doc) + if mail_doc.css(".yahoo_quoted") + extra_content = mail_doc.css(".yahoo_quoted")[0] + mail_doc.css(".yahoo_quoted")[0].remove + comment_content = mail_doc + else + comment_content = mail_doc + extra_content = nil + end + + { + "comment_content" => comment_content, + "extra_content" => extra_content + } + end + + def self.gmail_parsed_mail(mail_doc) + if mail_doc.css(".gmail_quote").any? + extra_content = mail_doc.css(".gmail_quote")[0] + mail_doc.css(".gmail_quote")[0].remove + comment_content = mail_doc + else + comment_content = mail_doc + extra_content = nil + end + + { + "comment_content" => comment_content, + "extra_content" => extra_content + } + end + + def trimmed_content? + comment.include?(COMMENT_FILTER) + end + end diff --git a/app/models/concerns/comments_shared.rb b/app/models/concerns/comments_shared.rb index 422ffe05183..9147782480c 100644 --- a/app/models/concerns/comments_shared.rb +++ b/app/models/concerns/comments_shared.rb @@ -2,10 +2,15 @@ # Refer to this link: http://stackoverflow.com/questions/14541823/how-to-use-concerns-in-rails-4 module CommentsShared extend ActiveSupport::Concern + include ApplicationHelper # filtered version additionally appending http/https # protocol to protocol-relative URLslike "/foo" def body_email(host = 'publiclab.org') + if contain_trimmed_body?(body) + comment_body = filtered_comment_body(body) + return comment_body.gsub(/([\s|"|'|\[|\(])(\/\/)([\w]?\.?#{host})/, '\1https://\3') + end body.gsub(/([\s|"|'|\[|\(])(\/\/)([\w]?\.?#{host})/, '\1https://\3') end diff --git a/app/views/notes/_comment.html.erb b/app/views/notes/_comment.html.erb index 334e6d32976..c5b7e6d3b1b 100644 --- a/app/views/notes/_comment.html.erb +++ b/app/views/notes/_comment.html.erb @@ -106,7 +106,15 @@
-

<%= raw render_comment_body(comment) %>

+ + <% comment_body = render_comment_body(comment) %> +

<%= raw filtered_comment_body(comment_body) %>

+ + <% if contain_trimmed_body?(comment_body) %> + +
<%= raw trimmed_body(comment_body) %>
+ <% end %> + <% if comment.body.include?('?') %>

Is this a question? Click here to post it to the Questions page.

diff --git a/test/fixtures/drupal_users.yml b/test/fixtures/drupal_users.yml index 14e721ead47..20bc6f64c63 100644 --- a/test/fixtures/drupal_users.yml +++ b/test/fixtures/drupal_users.yml @@ -74,3 +74,17 @@ legacy_user: status: 1 mail: legacy@publiclab.org created: <%= Time.now.to_i %> + +gmail_test: + name: namangupta + mail: 01namangupta@gmail.com + uid: 12 + status: 1 + created: <%= Time.now.to_i %> + +naman18996: + name: naman18996 + mail: naman18996@yahoo.com + uid: 13 + status: 1 + created: <%= Time.now.to_i %> diff --git a/test/fixtures/users.yml b/test/fixtures/users.yml index 9ff2a4ad5ba..74c57c27350 100644 --- a/test/fixtures/users.yml +++ b/test/fixtures/users.yml @@ -132,3 +132,29 @@ test_user: bio: '' created_at: <%= Time.now %> updated_at: <%= Time.now %> + +naman: + username: namangupta + status: 1 + email: 01namangupta@gmail.com + id: 12 + last_request_at: <%= Time.now %> + password_salt: <%= salt = Authlogic::Random.hex_token %> + crypted_password: <%= Authlogic::CryptoProviders::Sha512.encrypt("secretive" + salt) %> + persistence_token: <%= Authlogic::Random.hex_token %> + bio: '' + created_at: <%= Time.now %> + updated_at: <%= Time.now %> + +naman18996: + username: naman18996 + status: 1 + email: naman18996@yahoo.com + id: 13 + last_request_at: <%= Time.now %> + password_salt: <%= salt = Authlogic::Random.hex_token %> + crypted_password: <%= Authlogic::CryptoProviders::Sha512.encrypt("secretive" + salt) %> + persistence_token: <%= Authlogic::Random.hex_token %> + bio: '' + created_at: <%= Time.now %> + updated_at: <%= Time.now %> diff --git a/test/incoming_test_emails/gmail/final_parsed_comment.txt b/test/incoming_test_emails/gmail/final_parsed_comment.txt new file mode 100644 index 00000000000..1c5f2c8eb69 --- /dev/null +++ b/test/incoming_test_emails/gmail/final_parsed_comment.txt @@ -0,0 +1,11 @@ +This is another reply by email comment + +On Tue, Jul 3, 2018 at 11:17 PM Naman Gupta \<[01namangupta@gmail.com](mailto:01namangupta@gmail.com)\> wrote: + +> This is reply by comment +> +> +> On Tue, Jul 3, 2018 at 11:13 PM Rails Projects \<[railsprojects2018@gmail.com](mailto:railsprojects2018@gmail.com)\> wrote: +> +> > This is a comment sent to the user by publiclab. + diff --git a/test/incoming_test_emails/gmail/incoming_gmail_email.eml b/test/incoming_test_emails/gmail/incoming_gmail_email.eml new file mode 100644 index 00000000000..a2e496be5be --- /dev/null +++ b/test/incoming_test_emails/gmail/incoming_gmail_email.eml @@ -0,0 +1,107 @@ +Delivered-To: railsprojects2018@gmail.com +Received: by 2002:a1c:924f:0:0:0:0:0 with SMTP id u76-v6csp1316358wmd; + Tue, 3 Jul 2018 10:48:13 -0700 (PDT) +X-Received: by 2002:a50:e615:: with SMTP id y21-v6mr29798400edm.278.1530640093065; + Tue, 03 Jul 2018 10:48:13 -0700 (PDT) +ARC-Seal: i=1; a=rsa-sha256; t=1530640093; cv=none; + d=google.com; s=arc-20160816; + b=iwlu7+0DRR8CFpRnB6GQ1MSWcjrLdSomGi3xKTijlKdmBbwQrxmMpX6SNnzskE3xVC + SiylMtj5yTcxWmWTuKuOKLbprkSobBQ1vu+xRV1J7S28a/1nYOKEipCQKh3bgLl7lIqr + vRCnNZqt+afuol0O+97HmZAp3yYBmUSL6ArW+G5mId1Zxc25uV037xJeA/mKyr7FccW9 + D6W86z9XjMzlBUWB2k3V0vbsVJhBgxXxq22VJWx2ZmK4fTiyNsBqqjfjLyGFGdMVy/e6 + 9jvl66h3XmhJp49kWbWSlFk1RKGTXvDZJ4ZhwnBZ6wqUYtrXmwg85ibmdXxDCvYANvkc + JUbw== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; + h=to:subject:message-id:date:from:in-reply-to:references:mime-version + :dkim-signature:arc-authentication-results; + bh=RYGhy7yx79U1IqJDo1X6kis0HCj0a5eCbuu3JzStGEU=; + b=a3X4+9aFlZEKknmilS9b18oXGSTJxvMhh/nIqhFIL1fxYIdE7XCXY+TTLwrjTkWIo5 + XaghwYmv8vCedU2v0KfojMGaN6fkZXqXr+lcsYpRJqSUALoaYhJlAOl/bL54PzXewLZ6 + 5xrraJMyDdo/xBqd6P12Cndza5il6/kpiSHPdoGb/4F/zQOQ15ta0n8veXAGFTk3k4Yk + T9MrzB0ogcaFDYlJtPFRCHz4YenzAN3JZ0hmxwLnO7f0DfGvCFi35fAOFpxCCCzd4KzG + vhMcuS/LPCQNPUbN8NGRfSHAULgGtFNGL7PmG6FKFY+iPD10AL/zIFOlHoiLsJucgNE+ + hmpw== +ARC-Authentication-Results: i=1; mx.google.com; + dkim=pass header.i=@gmail.com header.s=20161025 header.b=plagx6JT; + spf=pass (google.com: domain of 01namangupta@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=01namangupta@gmail.com; + dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com +Return-Path: <01namangupta@gmail.com> +Received: from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41]) + by mx.google.com with SMTPS id k17-v6sor1127557edr.39.2018.07.03.10.48.13 + for + (Google Transport Security); + Tue, 03 Jul 2018 10:48:13 -0700 (PDT) +Received-SPF: pass (google.com: domain of 01namangupta@gmail.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41; +Authentication-Results: mx.google.com; + dkim=pass header.i=@gmail.com header.s=20161025 header.b=plagx6JT; + spf=pass (google.com: domain of 01namangupta@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=01namangupta@gmail.com; + dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=gmail.com; s=20161025; + h=mime-version:references:in-reply-to:from:date:message-id:subject:to; + bh=RYGhy7yx79U1IqJDo1X6kis0HCj0a5eCbuu3JzStGEU=; + b=plagx6JTlgloc5Mt7ji72zSoeeF46aFIv2TCnUbBW/qcTFIMyAiIgebAf1cR8YnkEx + 8taYLU3XfDwJMpSsqnyWxANkc7QqUQUSguvA9fVWq2HlUvULzPjhagsUyC28i2U+4P/V + lCu5YNJJD3skx5yCKQ8SgtVgJTQEYcJYMQII9lfdhOcgvN54NuttBEi8YhiTnFvelhcF + TvrQO/S2I8/djSLXVT9SId1UA9gSEdGScTnCS5j1+F7RQVjyE9wHnypAmv+oL4v2HhGD + YbCPjieeOnRYH80s9GQuY/S+eRaDp2M+hKoDo3wy7pkxqGgPN1LWm7by5AySjJyVhd0f + QtRQ== +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20161025; + h=x-gm-message-state:mime-version:references:in-reply-to:from:date + :message-id:subject:to; + bh=RYGhy7yx79U1IqJDo1X6kis0HCj0a5eCbuu3JzStGEU=; + b=pZSQnj3qWHgBs4OCiHQ0VXZ/Iovfx3KJym43xyzq2xi/Mp8GvsLNmR5XV7k3MLll2Z + WPb3MlLXHW8K04wESSWvHs5Wg85hjpphaUdGbqicxA61W2/vBZpdRL02p/wbuQ3Pu3HA + M8gASdw8PNJIXZNg053R7gxGGKnR3fW3fx1SbZCi88L1bT+Qihtx6gcI3Eq6gs6xSUa5 + p2V9JOlKI9CldZLH2ktxBSxfZJoMHCeDJBN720VRCkOpGH/Kmf2LRFuyCVSC7y4U9Ly9 + RmQchyy65WQqZKXZ6U+CCQZQ1FfSWjvjTCt0JfAkt/rJPgDzc6gzLD/vOgQAK3MD5R98 + W3DA== +X-Gm-Message-State: APt69E0SpKbIzAW22w+TY0xcx9Frrfnhb5xRiuXK430nqF2cSWvL1qtb vBWot7Yt6WZlROATEbXleNghfNTqUfcasRA12stohoZE +X-Google-Smtp-Source: AAOMgpf0oHAY3q9Mx0qWa/782HzUaDAgN/Ccf/jg0W9zN87UGy/WmbrLh2TARouOGNHrMLvcRb52bY0k8emcXvSLdBo= +X-Received: by 2002:a50:9493:: with SMTP id s19-v6mr29606779eda.285.1530640092632; Tue, 03 Jul 2018 10:48:12 -0700 (PDT) +MIME-Version: 1.0 +References: +In-Reply-To: +From: Naman Gupta <01namangupta@gmail.com> +Date: Tue, 3 Jul 2018 23:18:01 +0530 +Message-ID: +Subject: Re: New Comment on Note (#21) +To: railsprojects2018@gmail.com +Content-Type: multipart/alternative; boundary="000000000000001a4f05701beb69" + +--000000000000001a4f05701beb69 +Content-Type: text/plain; charset="UTF-8" + +This is another reply by email comment + +On Tue, Jul 3, 2018 at 11:17 PM Naman Gupta <01namangupta@gmail.com> wrote: + +> This is reply by comment +> +> On Tue, Jul 3, 2018 at 11:13 PM Rails Projects < +> railsprojects2018@gmail.com> wrote: +> +>> This is a comment sent to the user by publiclab. +>> +> + +--000000000000001a4f05701beb69 +Content-Type: text/html; charset="UTF-8" +Content-Transfer-Encoding: quoted-printable + +
This is another reply by email comment

On Tue, Jul 3, 2018 at 11:17 PM Naman Gup= +ta <01namangupta@gmail.com= +> wrote:
This i= +s reply by comment

On = +Tue, Jul 3, 2018 at 11:13 PM Rails Projects <railsprojects2018@gmail.com> w= +rote:
This is a co= +mment sent to the user by publiclab.
+
+
+ +--000000000000001a4f05701beb69-- \ No newline at end of file diff --git a/test/incoming_test_emails/gmail/incoming_gmail_email.html b/test/incoming_test_emails/gmail/incoming_gmail_email.html new file mode 100644 index 00000000000..97539db0ef6 --- /dev/null +++ b/test/incoming_test_emails/gmail/incoming_gmail_email.html @@ -0,0 +1,15 @@ +
This is another reply by email comment
+
+
+
On Tue, Jul 3, 2018 at 11:17 PM Naman Gupta <01namangupta@gmail.com> wrote:
+
+
This is reply by comment
+
+
+
On Tue, Jul 3, 2018 at 11:13 PM Rails Projects <railsprojects2018@gmail.com> wrote:
+
+
This is a comment sent to the user by publiclab.
+
+
+
+
\ No newline at end of file diff --git a/test/incoming_test_emails/gmail/readme.md b/test/incoming_test_emails/gmail/readme.md new file mode 100644 index 00000000000..fbd58258de9 --- /dev/null +++ b/test/incoming_test_emails/gmail/readme.md @@ -0,0 +1,9 @@ +This documents the various steps done to parse the mails. + +1. `incoming_gmail_email.eml` is a sample incoming mail form gmail. +2. `incoming_gmail_email.html` file containss `incoming_gmail_email.eml` converted to html +3. Finally `final_parsed_comment.txt` contains the final parsed comment. + +To parse the mails coming form gmail and to seperate the main body content and trimmed content which contains conversation thread information we have used a class `gmail_quote` which seperates the main body content and trimmed content. + +In `incoming_gmail_email.html` there is a class named `gmail_quote`, html element containing this class is the trimmed content. So to remove this we have used Nokogiri to parse html using which we have seperated the main body content and trimmed content based on class selector. diff --git a/test/incoming_test_emails/yahoo/final_parsed_comment.txt b/test/incoming_test_emails/yahoo/final_parsed_comment.txt new file mode 100644 index 00000000000..c752d86ee1b --- /dev/null +++ b/test/incoming_test_emails/yahoo/final_parsed_comment.txt @@ -0,0 +1,10 @@ +This is reply by mail comment from yahoooo + + On Tuesday, 3 July 2018, 11:20:57 PM IST, Rails Projects \ wrote: + + + + + +This is a comment sent to the user by publiclab. + diff --git a/test/incoming_test_emails/yahoo/incoming_yahoo_email.eml b/test/incoming_test_emails/yahoo/incoming_yahoo_email.eml new file mode 100644 index 00000000000..7ef10cdf532 --- /dev/null +++ b/test/incoming_test_emails/yahoo/incoming_yahoo_email.eml @@ -0,0 +1,79 @@ +Delivered-To: railsprojects2018@gmail.com +Received: by 2002:a1c:924f:0:0:0:0:0 with SMTP id u76-v6csp1339484wmd; + Tue, 3 Jul 2018 11:13:08 -0700 (PDT) +X-Google-Smtp-Source: ADUXVKLsANtG+QeoJ0mvxwnjZMzn18uVV/0+ihoGm0SwbKz3mS15McYnROOLljNq5EvWMsiTWN5X +X-Received: by 2002:a17:902:8d91:: with SMTP id v17-v6mr31583874plo.9.1530641588321; + Tue, 03 Jul 2018 11:13:08 -0700 (PDT) +ARC-Seal: i=1; a=rsa-sha256; t=1530641588; cv=none; + d=google.com; s=arc-20160816; + b=pVnuGBSUMiy//7v2MEBxQhGHS6T3l2DgOl/Y9xn+ls8qBCydLIOf/zEbfKNhLkfRLE + kdE3gtaxq2//TDmJWDolfwMd9mDWC7pjB2iP85dY2mRCyzbczwnU9Sc+JJrtEoRkQgwg + s95y4loLsR0vZaLu1jsOcvpwhxZ0HCuqOuhrog37SsGklPynusRj3uCFmIuAqUu3tl/Z + +LniU4efPNMxMD3oTafkZfziiRwL8TE028oGk8Ch2R7L0+yeHTOqgmNiLccmLtUC8iK3 + TGyG2WXYDvbwUdoe6BYkWaPvYeLHNEjMdOqR+z3v4lyLeIOGrLzLsnxVI9lseCZ7EsZz + Yqyw== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; + h=mime-version:subject:references:in-reply-to:message-id:to:from:date + :dkim-signature:arc-authentication-results; + bh=ddwESwcamoc9p/2wVtdLGmAUUIZeLlBkH/vdAIMLwOo=; + b=SQJknL54sS19gHGVMaZF4pr0wWeTpFkWR/tjmvSAL/YeKzNetbgSH9JsNVitbxH64i + BCQquotID3KofgmwyYhPQ6NuwKkPtQ3V0xnKHolTbNLRXdgWCSHeSXzBQYDybPWmtGmd + Wg4D2UJK6fNm0m1OYYSFxfgYLnsce6QzBuyin59A6Uuq6401HiwgUTrljulR/Y+tGEBn + LFeVqoDXd+wS9SbMX6br3PQYbOXOF+joz5c5PyBch7Hyn/LKD/cFhPtzTUmW+TLE6IFV + z7gS98gZDOH9De7h5/rwVE0Ya+pipSeg+XvP0LuQmJ9Qdo6qzmjeQ1Xe+knGBPy/u3+T + elTQ== +ARC-Authentication-Results: i=1; mx.google.com; + dkim=pass header.i=@yahoo.com header.s=s2048 header.b=b8bBTNjc; + spf=pass (google.com: domain of naman18996@yahoo.com designates 106.10.242.168 as permitted sender) smtp.mailfrom=naman18996@yahoo.com; + dmarc=pass (p=REJECT sp=REJECT dis=NONE) header.from=yahoo.com +Return-Path: +Received: from sonic302-48.consmr.mail.sg3.yahoo.com (sonic302-48.consmr.mail.sg3.yahoo.com. [106.10.242.168]) + by mx.google.com with ESMTPS id u3-v6si1498029pgq.354.2018.07.03.11.13.07 + for + (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); + Tue, 03 Jul 2018 11:13:08 -0700 (PDT) +Received-SPF: pass (google.com: domain of naman18996@yahoo.com designates 106.10.242.168 as permitted sender) client-ip=106.10.242.168; +Authentication-Results: mx.google.com; + dkim=pass header.i=@yahoo.com header.s=s2048 header.b=b8bBTNjc; + spf=pass (google.com: domain of naman18996@yahoo.com designates 106.10.242.168 as permitted sender) smtp.mailfrom=naman18996@yahoo.com; + dmarc=pass (p=REJECT sp=REJECT dis=NONE) header.from=yahoo.com +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; t=1530641586; bh=ddwESwcamoc9p/2wVtdLGmAUUIZeLlBkH/vdAIMLwOo=; h=Date:From:To:In-Reply-To:References:Subject:From:Subject; b=b8bBTNjchM7ONQQSFLWWDzAyt44kLP08DvAlZrXRrcH1CSJnu6mmMIsRn3OrlFDBpuu+9ObBggk3r92YEHNKMYuhSzYgJ3KgqS+2kZCmmPRK+vtsZDHps7RITE4ydyoJeL+LIUOUAJR9gQ+m9FpHwQxmrUNLlC/HGkmxCEktITiJNqp8YGCvbsaU4vtIT6Bf3hzxJUgF4NnYhimdShhum+vGuEowZf65rFGaa4eyJjeink84sHLH2+Ul1ML/uvOJZeiQE3mnlMg09lXoe7ofINO43JnKSWNrEQDPlMt+z7eUun/P+M9rS+PA7fqgXqWN/wEMPmqivqUUvdNYXpOiYw== +X-YMail-OSG: GOoAr_UVM1k156EiiTItEtvXqzQsyP8kmI.62mDQrscGF.JWV45AuZI2McDmxuG KSs8ywZFPJGGNNkM.Nu_Etqy2qQvKoVPCrQAx7aHstRLUTFSrPOLtnhyV_CYZn7HpOzezt3z0Q1Y RI6wDDcvc5mjO8BYhjdJlq0.EvVaytitDgXH_cZZ0EYkE_tA_6Ydk4Eoe2xxBDrYhr_o2XLW5yb2 FmTYxIiR5xuZ5M8_HDSZ29240OL2db2rF.h7UYuNyggeO7OjLHFJM4AINgsPGE0odjUZr.U.L7H1 CduwiyPH6nIXcylh6NB87RMLDTvfZB_QaamOTHd2cwAdaQd4fA.e2Mun5dcG6EMR4MwQzgwO9Ojd gFvt1YTEEVr5oiNdudoIVgsfO913aVtavztJfPwgL7F_jnCH9rUWW2lzfW2OduEvElxE_drWh5Cc GyQsajT1KjpIGq.SoQAh4QDNIAgujUZbsb8kuGbWKA_FHtXC3Tp55GmnRURzmAKBSJ7X0MotksqK kkvt0kvbJ821Uh2cTwaVDNetjJAZcAbdBz8OFQ4dmuEmq.f_7o71PToRZwt738uyCBjo_LzpMmgm oO5lMZS3Vz9sUIZEr2vBB7Mz59AdbEicdHTHzmKReNxcxazXSBSWS067vf.zUOMkO1q8YTbNw_zh 6JEbnr7FhcWjZTMBkJPhGSG0gsVLFMfbSEDnOvkh8ZD9ylb7i7bUpDBsODhn79nDsicn5AohTlcS qv5YXSzlL.N4TMJiDXOfaka_qyqsSZGsmVSxBdslMf_Wa1uAJeQonjUDgXcWup07vn_OSJn2AquR WTXICr2iz3shpdEz1jETDvbT01TgvNWo4hORqsQ.JuMr5qHO76q_E.YrCbYGJ.dUPyLWo3uTD60C bVa1zL.GQw8wAZ57g2qpimibEkAyzcn_ooAcZOBYO4fJt1NIx99hdNms6Dd7iX.eUDB33cJEMwQr X +Received: from sonic.gate.mail.ne1.yahoo.com by sonic302.consmr.mail.sg3.yahoo.com with HTTP; Tue, 3 Jul 2018 18:13:06 +0000 +Date: Tue, 3 Jul 2018 18:09:05 +0000 (UTC) +From: Naman Gupta +To: Rails Projects +Message-ID: <268274679.2142643.1530641345720@mail.yahoo.com> +In-Reply-To: +References: +Subject: Re: New Comment on Note (#21) +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="----=_Part_2142642_1211954234.1530641345718" +X-Mailer: WebService/1.1.12062 YMailNorrin Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8 +Content-Length: 1343 + +------=_Part_2142642_1211954234.1530641345718 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +This is reply by mail comment from yahoooo On Tuesday, 3 July 2018, 11:20:57 PM IST, Rails Projects wrote: + + This is a comment sent to the user by publiclab. + +------=_Part_2142642_1211954234.1530641345718 +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +
This is reply by mail comment from yahoooo
+
+ +
+ On Tuesday, 3 July 2018, 11:20:57 PM IST, Rails Projects <railsprojects2018@gmail.com> wrote: +
+

+

+
This is a comment sent to the user by publiclab.
+
+
+
+------=_Part_2142642_1211954234.1530641345718-- \ No newline at end of file diff --git a/test/incoming_test_emails/yahoo/incoming_yahoo_mail.html b/test/incoming_test_emails/yahoo/incoming_yahoo_mail.html new file mode 100644 index 00000000000..b98c99bc1d1 --- /dev/null +++ b/test/incoming_test_emails/yahoo/incoming_yahoo_mail.html @@ -0,0 +1,22 @@ + + + +
+
This is reply by mail comment from yahoooo
+
+
+
+ On Tuesday, 3 July 2018, 11:20:57 PM IST, Rails Projects <railsprojects2018@gmail.com> wrote: +
+

+

+
+
+
This is a comment sent to the user by publiclab.
+
+
+
+
+
+ + \ No newline at end of file diff --git a/test/incoming_test_emails/yahoo/readme.md b/test/incoming_test_emails/yahoo/readme.md new file mode 100644 index 00000000000..debd67e5542 --- /dev/null +++ b/test/incoming_test_emails/yahoo/readme.md @@ -0,0 +1,9 @@ +This documents the various steps done to parse the mails. + +1. `incoming_yahoo_email.eml` is a sample incoming mail form yahoo. +2. `incoming_yahoo_email.html` file containss `incoming_yahoo_email.eml` converted to html +3. Finally `final_parsed_comment.txt` contains the final parsed comment. + +To parse the mails coming form yahoo and to seperate the main body content and trimmed content which contains conversation thread information we have used a class `yahoo_quoted` which seperates the main body content and trimmed content. + +In `incoming_yahoo_email.html` there is a class named `yahoo_quoted`, html element containing this class is the trimmed content. So to remove this we have used Nokogiri to parse html using which we have seperated the main body content and trimmed content based on class selector. diff --git a/test/unit/comment_test.rb b/test/unit/comment_test.rb index 7ca3bfb4ce5..922ef8b0d28 100644 --- a/test/unit/comment_test.rb +++ b/test/unit/comment_test.rb @@ -1,5 +1,4 @@ require 'test_helper' - class CommentTest < ActiveSupport::TestCase test 'should save comment' do comment = Comment.new @@ -222,4 +221,28 @@ class CommentTest < ActiveSupport::TestCase map = comment.user_reactions_map assert_equal map["Heart"], "Bob and jeff reacted with heart emoji" end + + + test 'should parse incoming mail from gmail service correctly' do + require 'mail' + mail = Mail.read('test/incoming_test_emails/gmail/incoming_gmail_email.eml') + node = Node.last + mail.subject = "Re: #{node.title} (##{node.nid})" + Comment.receive_mail(mail) + f = File.open('test/incoming_test_emails/gmail/final_parsed_comment.txt', 'r') + comment = Comment.last + assert_equal comment.comment, f.read + end + + test 'should parse incoming mail from yahoo service correctly' do + require 'mail' + mail = Mail.read('test/incoming_test_emails/yahoo/incoming_yahoo_email.eml') + node = Node.last + mail.subject = "Re: #{node.title} (##{node.nid})" + Comment.receive_mail(mail) + f = File.open('test/incoming_test_emails/yahoo/final_parsed_comment.txt', 'r') + comment = Comment.last + assert_equal comment.comment, f.read + end + end