Add article.comment_score metric calc and tracking (forem#5560) [deploy]

* Add article.comment_score metric calc and tracking

* Change from gauge to count

* Adjust key and tags

app/jobs/articles/score_calc_job.rb

@@ -7,6 +7,7 @@ def perform(article_id)
       return unless article
 
       article.update_columns(score: article.reactions.sum(:points),
+                             comment_score: article.comments.sum(:score),
                              hotness_score: BlackBox.article_hotness_score(article),
                              spaminess_rating: BlackBox.calculate_spaminess(article))
       article.index!

app/workers/metrics/record_daily_usage_worker.rb

@@ -4,21 +4,31 @@ class RecordDailyUsageWorker
     sidekiq_options queue: :low_priority, retry: 10
 
     def perform
+      # Articles published in the past 24 hours with at least 15 "score" (positive/negative reactions)
       articles_min_15_score_past_24h = Article.published.where("score >= ? AND published_at > ?", 15, 1.day.ago).size
       DataDogStatsClient.count("articles.min_15_score_past_24h", articles_min_15_score_past_24h, tags: { resource: "articles" })
 
+      # Articles published in the past 24 hours with at least 15 "score" (positive/negative reactions)
+      articles_min_15_comment_score_past_24h = Article.published.where("comment_score >= ? AND published_at > ?", 15, 1.day.ago).size
+      DataDogStatsClient.count("articles.min_15_comment_score_past_24h", articles_min_15_comment_score_past_24h, tags: { resource: "articles" })
+
+      # Articles published in the past 24 which were that user's first article
       first_articles_past_24h = Article.where(nth_published_by_author: 1).where("published_at > ?", 1.day.ago).size
       DataDogStatsClient.count("articles.first_past_24h", first_articles_past_24h, tags: { resource: "articles" })
 
+      # Users who signed up in the past 24 hours who have made at least 1 comment so far
       new_users_min_1_comment_past_24h = User.where("comments_count >= ? AND created_at > ?", 1, 24.hours.ago).size
       DataDogStatsClient.count("users.new_min_1_comment_past_24h", new_users_min_1_comment_past_24h, tags: { resource: "users" })
 
+      # Total negative reactions in the past 24 hours
       nagative_reactions_past_24h = Reaction.where("points < 0").where("created_at > ?", 1.day.ago).size
       DataDogStatsClient.count("reactions.negative_past_24h", nagative_reactions_past_24h, tags: { resource: "reactions" })
 
+      # Total abuse (etc.) reports in the past 24 hours
       reports_past_24_hours = FeedbackMessage.where(category: ["spam", "other", "rude or vulgar", "harassment"]).where("created_at > ?", 1.day.ago).size
       DataDogStatsClient.count("feedback_messages.reports_past_24_hours", reports_past_24_hours, tags: { resource: "feedback_messages" })
 
+      # Counts of total days active (1-7) in the past week, e.g. 5000 users visited once this week, 3500 visited twice, etc.
       get_days_active_past_week_counts
     end
 
@@ -29,8 +39,9 @@ def get_days_active_past_week_counts
       7.times do |i|
         ids_by_day << PageView.where("created_at > ? AND created_at < ?", (i + 1).days.ago, i.days.ago).where.not(user_id: nil).pluck(:user_id).uniq
       end
-      non_new_user_ids = User.where("created_at < ?", 7.days.ago).where(id: ids_by_day.flatten).pluck(:id)
-      new_user_ids = User.where("created_at > ?", 7.days.ago).where(id: ids_by_day.flatten).pluck(:id)
+      flat_id_list = ids_by_day.flatten.uniq
+      non_new_user_ids = User.where("created_at < ?", 7.days.ago).where(id: flat_id_list).pluck(:id)
+      new_user_ids = User.where("created_at > ? AND created_at < ?", 8.days.ago, 7.days.ago).where(id: flat_id_list).pluck(:id)
       record_active_days_of_group(ids_by_day, non_new_user_ids, "established")
       record_active_days_of_group(ids_by_day, new_user_ids, "new")
     end
@@ -40,7 +51,7 @@ def record_active_days_of_group(ids_by_day, user_ids, group)
       distinct_user_values = user_ids_by_day.flatten.group_by(&:itself).transform_values(&:count).values
       distinct_counts = distinct_user_values.group_by(&:itself).transform_values(&:count)
       distinct_counts.keys.each do |key|
-        DataDogStatsClient.gauge("users.#{group}_active_#{key}_days_past_7_days", distinct_counts[key], tags: { resource: "users" })
+        DataDogStatsClient.count("users.active_days_past_week", distinct_counts[key], tags: { resource: "users", group: group, day_count: key })
       end
     end
   end

db/migrate/20200117135558_add_comment_score_to_articles.rb

@@ -0,0 +1,5 @@
+class AddCommentScoreToArticles < ActiveRecord::Migration[5.2]
+  def change
+    add_column :articles, :comment_score, :integer, default: 0
+  end
+end

db/migrate/20200117135902_add_comment_score_index_to_articles.rb

@@ -0,0 +1,7 @@
+class AddCommentScoreIndexToArticles < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :articles, :comment_score, algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -10,7 +10,8 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_12_27_114543) do
+ActiveRecord::Schema.define(version: 2020_01_17_135902) do
+
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -67,6 +68,7 @@
     t.string "canonical_url"
     t.integer "collection_id"
     t.integer "collection_position"
+    t.integer "comment_score", default: 0
     t.string "comment_template"
     t.integer "comments_count", default: 0, null: false
     t.datetime "created_at", null: false
@@ -136,6 +138,7 @@
     t.string "video_state"
     t.string "video_thumbnail_url"
     t.index ["boost_states"], name: "index_articles_on_boost_states", using: :gin
+    t.index ["comment_score"], name: "index_articles_on_comment_score"
     t.index ["featured_number"], name: "index_articles_on_featured_number"
     t.index ["feed_source_url"], name: "index_articles_on_feed_source_url"
     t.index ["hotness_score"], name: "index_articles_on_hotness_score"

spec/jobs/articles/score_calc_job_spec.rb

@@ -11,6 +11,8 @@
 
     context "with article" do
       let_it_be(:article) { create(:article) }
+      let_it_be(:comment) { create(:comment, commentable: article, score: 5) }
+      let_it_be(:second_comment) { create(:comment, commentable: article, score: 7) }
 
       it "updates article scores", :aggregate_failures do
         allow(Article).to receive(:find_by).and_return(article)
@@ -20,6 +22,7 @@
         article.reload
 
         expect(article.score).to be(7)
+        expect(article.comment_score).to be(12)
         expect(article.hotness_score).to be(373)
         expect(article.spaminess_rating).to be(2)
       end

spec/workers/metrics/record_daily_usage_worker_spec.rb

@@ -5,9 +5,9 @@
   let_it_be(:first_user) { create(:user, comments_count: 1) }
   let_it_be(:second_user) { create(:user, comments_count: 2) }
   let_it_be(:third_user) { create(:user, comments_count: 0) }
-  let_it_be(:first_article) { create(:article, score: 15, nth_published_by_author: 1) }
-  let_it_be(:second_article) { create(:article, score: 5, nth_published_by_author: 2) }
-  let_it_be(:third_article) { create(:article, score: 38, nth_published_by_author: 3) }
+  let_it_be(:first_article) { create(:article, score: 15, nth_published_by_author: 1, comment_score: 25) }
+  let_it_be(:second_article) { create(:article, score: 5, nth_published_by_author: 2, comment_score: 0) }
+  let_it_be(:third_article) { create(:article, score: 38, nth_published_by_author: 3, comment_score: 2) }
   let_it_be(:user) { create(:user, :trusted) }
   let_it_be(:reaction) { create(:reaction, category: "vomit", user: user, reactable: first_article) }
   let_it_be(:feedback_message) { create(:feedback_message, :abuse_report) }
@@ -23,6 +23,12 @@
       ).to have_received(:count).with("articles.min_15_score_past_24h", 2, Hash).at_least(1)
     end
 
+    it "logs articles with at least comment 15 score" do
+      expect(
+        DataDogStatsClient,
+      ).to have_received(:count).with("articles.min_15_comment_score_past_24h", 1, Hash).at_least(1)
+    end
+
     it "records first articles" do
       expect(
         DataDogStatsClient,