init measurement protol

add /debug/mp/collect/:site_uuid

init debug mp schema

init measurement protocol model

add redis client

check api_secret

add mp testing

rm useless

measurement protocol job

fix ci

check api_secret for job

app/controllers/measurement_protocols_controller.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class MeasurementProtocolsController < ApplicationController
+  before_action :get_site
+
+  def create
+    @mp = @site.measurement_protocols.create
+    redirect_to site_measurement_protocols_path(@site)
+  end
+
+  def destroy
+    @mp = @site.measurement_protocols.find(params[:id])
+    @mp.destroy
+    redirect_to site_measurement_protocols_path(@site)
+  end
+
+  def index
+    @mps = @site.measurement_protocols
+  end
+
+  def get_site
+    @site = current_user.sites.find_by!(uuid: params[:site_id])
+  end
+end

app/helpers/application_helper.rb

@@ -112,6 +112,7 @@ def render_setting_nav_css(tab_name)
     none_current = "text-gray-600 dark:text-gray-400 rounded-md hover:text-gray-900 dark:hover:text-gray-100 hover:bg-gray-50 dark:hover:bg-gray-800 outline-none focus:outline-none focus:text-gray-900 focus:bg-gray-50 dark:focus:text-gray-100 dark:focus:bg-gray-800"
     return current if params[:controller] == "settings" && params[:action] == "general" && tab_name.to_s == "general"
     return current if params[:controller] == "site_connections" && params[:action] == "index" && tab_name.to_s == "site_connection"
+    return current if params[:controller] == "measurement_protocols" && params[:action] == "index" && tab_name.to_s == "measurement_protocol"
 
     none_current
   end

app/jobs/mp_event_job.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class MpEventJob
+  include Sidekiq::Worker
+
+  COLUMN_NAMES = Hyper::Event.column_names
+
+  def perform(*args)
+    Thread.current[:bqmp] ||= BufferQueue.new(max_batch_size: (ENV["MAX_BATCH_SIZE"] || 50).to_i, execution_interval: (ENV["EXECUTION_INTERVAL"] || 5).to_i) do |batch|
+      puts "bulk insert #{batch.size} records"
+      Hyper::Event.insert_all!(batch)
+    end
+    params, form, request = args
+
+    api_secret = params["api_secret"]
+    return if api_secret.nil?
+    _firebase_app_id = params["firebase_app_id"]
+
+    meta_path = request["path"].match(/\/(?<uuid>.*?)\/mp\/collect/)
+    site_uuid = meta_path["uuid"]
+    return if site_uuid.nil?
+    return unless RedisClient.get(api_secret) == site_uuid
+    payload = JSON.parse(form)
+    events = payload["events"]
+    _app_instance_id = payload["app_instance_id"]
+    client_id = payload["client_id"]
+    user_id = payload["user_id"]
+    _time = payload["timestamp_micros"]
+    user_props = payload["user_properties"]
+    tracking_id = params["measurement_id"]
+
+    events.each do |event|
+      tech_info = TechDetector.detect(request["user_agent"])
+      tech_info.delete(:is_bot)
+      real_ip = request["x_forwarded_for"] || request["ip"]
+      ip_info = IPDB.get(real_ip)
+
+      referrer_source = RefererSourceDetector.detect(params["dr"])
+      traffic_info = TrafficDetector.detect({ "dl" => params["dl"], "dr" => params["dr"] })
+
+      result = {}
+      result.merge!(traffic_info)
+      result.merge!(tech_info)
+      result.merge!({
+        site_id: site_uuid,
+        event_name: event["name"],
+        session_id: client_id,
+        client_id: client_id,
+        user_id: user_id,
+        tracking_id: tracking_id,
+        started_at: Time.now,
+
+        protocol_version: "2",
+        data_source: "web",
+
+        title: params["dt"],
+        user_agent: request["user_agent"],
+        ip: real_ip,
+        referrer: params["dr"],
+        referrer_source: referrer_source,
+        screen_resolution: params["sr"],
+        user_language: params["ul"],
+
+        country: ip_info[:country_name],
+        city: ip_info[:city],
+        region: ip_info[:region_name],
+        latitude: ip_info[:latitude],
+        longitude: ip_info[:longitude],
+
+        user_props: user_props,
+        event_props: event["params"],
+
+        raw_event: event
+      })
+      Thread.current[:bqmp].push result
+    end
+  end
+end

app/models/measurement_protocol.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: measurement_protocols
+#
+#  id         :bigint           not null, primary key
+#  api_secret :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  site_id    :bigint
+#
+# Indexes
+#
+#  index_measurement_protocols_on_site_id  (site_id)
+#
+class MeasurementProtocol < ApplicationRecord
+  belongs_to :site
+
+  before_create do
+    self.api_secret = SecureRandom.hex(20)
+  end
+
+  after_create do
+    RedisClient.set(api_secret, site.uuid)
+  end
+
+  before_destroy do
+    RedisClient.del(api_secret)
+  end
+end

app/models/site.rb

@@ -29,6 +29,7 @@ class Site < ApplicationRecord
 
   has_many :shared_links
   has_many :site_connections
+  has_many :measurement_protocols
 
   validates :domain, presence: true
   validates :tracking_id, presence: true

app/views/measurement_protocols/index.html.erb

@@ -0,0 +1,43 @@
+<div class="container pt-6">
+  <a class="text-sm text-indigo-600 font-bold" href="<%= site_path(@site) %>">← Back to stats</a>
+  <div class="pb-5 border-b border-gray-200 dark:border-gray-500">
+    <h2 class="text-2xl font-bold leading-7 text-gray-900 dark:text-gray-100 sm:text-3xl sm:leading-9 sm:truncate">
+      Settings for <%= @site.domain %>
+    </h2>
+  </div>
+  <div class="lg:grid lg:grid-cols-12 lg:gap-x-5 lg:mt-4">
+    <%= render partial: 'settings/nav' %>
+
+    <div class="space-y-6 lg:col-span-9 lg:mt-4">
+      <div class="shadow bg-white dark:bg-gray-800 sm:rounded-md sm:overflow-hidden py-6 px-4 sm:p-6">
+        <header class="relative">
+          <h2 class="text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">Measurement Protocols</h2>
+          <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">Create a api secret to enable additional events to be sent into this site through the Measurement Protocol.</p>
+          <a href="" target="_blank">
+                <svg class="w-6 h-6 absolute top-0 right-0 text-gray-400" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" d="M18 10a8 8 0 11-16 0 8 8 0 0116 0zm-7-4a1 1 0 11-2 0 1 1 0 012 0zM9 9a1 1 0 000 2v3a1 1 0 001 1h1a1 1 0 100-2v-3a1 1 0 00-1-1H9z" clip-rule="evenodd"></path></svg>
+          </a>
+        </header>
+
+        <div class="mt-6">
+          <% @mps.each do |conn| %>
+            <div class="flex relative w-full max-w-xl mt-2 text-sm">
+              <input type="text" id="<%= conn.id %>" readonly="readonly" value="<%= conn.api_secret %>" class="transition bg-gray-100 dark:bg-gray-900 outline-none appearance-none border border-transparent rounded rounded-r-none w-full p-2 text-gray-700 dark:text-gray-300 focus:outline-none focus:border-gray-300 dark:focus:border-gray-500">
+              <button onclick="var input = document.getElementById('<%= conn.id %>'); input.focus(); input.select(); document.execCommand('copy');" href="javascript:void(0)" class="py-2 px-4 bg-gray-200 dark:bg-gray-850 text-indigo-800 dark:text-indigo-500 rounded-none border-r border-gray-300 dark:border-gray-500 hover:bg-gray-300 dark:hover:bg-gray-825">
+                <svg class="feather-sm" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path></svg>
+              </button>
+              <%= link_to site_measurement_protocol_path(@site, conn), method: :delete, type: :button, remote: true, data: {confirm: "sure?"}, class: "py-2 px-4 bg-gray-200 dark:bg-gray-850 text-red-600 dark:text-red-500 rounded-l-none hover:bg-gray-300 dark:hover:bg-gray-825" do %>
+                <svg class="feather feather-sm" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"></polyline><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"></path><line x1="10" y1="11" x2="10" y2="17"></line><line x1="14" y1="11" x2="14" y2="17"></line></svg>
+              <% end %>
+
+
+            </div>
+          <% end %>
+
+          <%= link_to "+ New", site_measurement_protocols_path(@site), remote: true, method: :post, class: "button mt-4", data: {disable_with: disable_with_spinner('+ New')}  %>
+
+        </div>
+      </div>
+
+    </div>
+  </div>
+</div>

app/views/settings/_nav.html.erb

@@ -1,12 +1,10 @@
 <div class="py-4 g:py-0 lg:col-span-3">
   <form action="" class="lg:hidden" method="post">
-    <select class="dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" id="tab" name="tab" onchange="location.href = location.href.replace(/[^//]*$/, event.target.value)">
-      <option value="general" selected="">General</option>
-      <option value="visibility">Visibility</option>
-      <option value="goals">Goals</option>
-      <option value="search_console">Search Console</option>
-      <option value="email_reports">Email reports</option>
-      <option value="danger_zone">Danger zone</option></select>
+    <select class="dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" id="tab" name="tab" onchange="location.href = event.target.value">
+      <option value="<%= general_site_settings_path(@site)  %>" selected="">General</option>
+      <option value="<%= site_site_connections_path(@site) %>">Site Connection</option>
+      <option value="<%= site_measurement_protocols_path(@site) %>">Measurement Protocol</option>
+    </select>
   </form>
   <div class="hidden lg:block">
 
@@ -16,6 +14,8 @@
 
     <a href="<%= site_site_connections_path(@site) %>" class="flex items-center px-3 py-2 text-sm leading-5 font-medium  transition ease-in-out duration-150  <%= render_setting_nav_css(:site_connection) %> mt-1">Connection</a>
 
+    <a href="<%= site_measurement_protocols_path(@site) %>" class="flex items-center px-3 py-2 text-sm leading-5 font-medium  transition ease-in-out duration-150  <%= render_setting_nav_css(:measurement_protocol) %> mt-1">Measurement Protocol</a>
+
 
 
   </div>

config/application.rb

@@ -44,6 +44,6 @@ class Application < Rails::Application
     }
 
     config.action_mailer.delivery_method = :smtp
-    # config.logger = ActiveSupport::Logger.new(STDOUT)
+    config.logger = ActiveSupport::Logger.new(STDOUT)
   end
 end

config/initializers/redis.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+RedisClient = Redis.new(driver: :hiredis, url: ENV["SAAS_REDIS_URL"] || ENV["REDIS_URL"] || "redis://localhost:6379/0")
+
+MeasurementProtocol.all.each do |mp|
+  RedisClient.set(mp.api_secret, mp.site.uuid)
+end

config/initializers/traffic_detector.rb

@@ -14,6 +14,8 @@ class TrafficDetector
   def self.detect(event)
     result = {}
     request_url = event["dl"]
+    return result if request_url.nil?
+
     request_uri = Addressable::URI.parse(request_url).normalize
     request_params = request_uri.query_values || {}
 

config/routes.rb

@@ -22,6 +22,7 @@
     resources :oses, only: [:index]
 
     resources :site_connections, only: [:index, :new, :show, :destroy, :create]
+    resources :measurement_protocols, only: [:index, :new, :show, :destroy, :create]
     member do
       get "snippet"
       get "debug"

db/migrate/20210404121858_create_measurement_protocols.rb

@@ -0,0 +1,9 @@
+class CreateMeasurementProtocols < ActiveRecord::Migration[6.1]
+  def change
+    create_table :measurement_protocols do |t|
+      t.bigint :site_id, index: true
+      t.string :api_secret, null: false
+      t.timestamps
+    end
+  end
+end