cool

app/models/spider_pages.rb

@@ -11,4 +11,17 @@ def get_parent_name
     end
   end
 
+  def get_category(user_id)
+    c = Category.find_by_name_and_user_id(self.name, user_id)
+    unless c
+      c = Category.new
+      c.user_id = user_id
+      c.parent_id = self.parent_id
+      c.name = self.name
+      c.save
+    end
+    c 
+  end
+
+
 end

app/models/user.rb

@@ -1092,6 +1092,27 @@ def is_shop
     (self.apply_type.to_i > 0 )
   end
 
+  def self.create_by_brand(brand_id)
+    brand = Brand.find_by_id brand_id
+    if brand
+      u = User.find_by_brand_id(brand.id)
+      unless u
+        username = brand.url + Devise.friendly_token[0,20]
+        email = username+"@dapeimishu.com"
+        password = Devise.friendly_token[0,20]
+        u = User.new
+        u.name = brand.display_name
+        u.email = email
+        u.password = password
+        u.brand_id = brand.id
+        u.remember_me = 1
+        u.save 
+      end
+      u
+    end
+  end
+
+
   private
   def get_pod_url
     pod_url = AppConfig[:pod_url].dup

db/migrate/20141220074003_add_page_id_to_matters.rb

@@ -0,0 +1,5 @@
+class AddPageIdToMatters < ActiveRecord::Migration
+  def change
+    add_column :matters, :page_id, :integer
+  end
+end

db/migrate/20141220102114_add_user_id_to_category.rb

@@ -0,0 +1,5 @@
+class AddUserIdToCategory < ActiveRecord::Migration
+  def change
+    add_column :categories, :user_id, :integer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20141216103608) do
+ActiveRecord::Schema.define(:version => 20141220102114) do
 
   create_table "all_tags", :force => true do |t|
     t.string   "name"
@@ -177,6 +177,7 @@
     t.string   "thumb_url",                      :default => ""
     t.float    "min_price"
     t.float    "max_price"
+    t.integer  "user_id"
   end
 
   add_index "categories", ["abb"], :name => "index_categories_on_abb"
@@ -750,6 +751,7 @@
     t.string   "docid"
     t.integer  "off_percent"
     t.integer  "origin_price"
+    t.integer  "page_id"
   end
 
   add_index "matters", ["brand_id"], :name => "index_matters_on_brand_id"

lib/tasks/update_matter.rake

@@ -35,88 +35,6 @@ def make_url(url, pno)
   url
 end
 
-def make_doc(doc)
-    ndoc = {}
-    ndoc[:docid] = make_docid( doc["url"] )
-    ndoc[:from]  = "homepage"
-    ndoc[:title] = doc["title"]
-    if ndoc[:title] == ""
-      ndoc[:title] = "新品"
-    end
-
-    doc_price = doc["price"] || "" #.match(/[\d\.,]+/).to_s if doc["price"]
-    if doc_price =~ /美元/
-      doc_price = doc_price.gsub(/美元/, "$")
-    end
-    if doc_price =~ /美西价：/
-      doc_price = doc_price.gsub(/美西价：/, "")
-    end
-    ndoc[:price] = doc_price
-    doc_origin_price = doc["origin_price"] || ""
-    if doc_origin_price =~ /美西价：/
-      doc_origin_price = doc_origin_price.gsub(/美西价：/, "")
-    end
-    ndoc[:origin_price] = doc_origin_price.strip.empty? ? doc_price : doc_origin_price
-    ndoc[:desc]  = doc["desc"]
-    ndoc[:brand_id] = doc["brand_id"]
-    if brand = Brand.find(doc["brand_id"])
-      ndoc[:level] = brand.level
-    end     
-    ndoc[:sold_count] = doc["sold_count"] unless doc["sold_count"].nil? || doc["sold_count"].strip.empty?
-    ndoc[:comments_count] = doc["hot"] unless doc["hot"].nil? || doc["hot"].strip.empty?
-    ndoc[:rate_score] = doc["rate"] unless doc["rate"].nil? || doc["rate"].strip.empty?
-    unless doc["store_count"].nil? || doc["store_count"].strip.empty?
-      store_count_val = doc["store_count"].match(/\d+/).to_s
-      ndoc[:store_count] = store_count_val
-    end
-
-    if ndoc[:brand_id] == "18"
-      if ndoc[:title].downcase.index("shoebox")
-         ndoc[:brand] = "shoebox"
-         ndoc[:brand_id] = "1035"
-      end
-    end
-    ndoc[:publish] = doc["time"]
-    url = doc["url"]
-    pno = doc["pno"] || ""
-    buy_url = make_url(url, pno)
-    ndoc[:buy_url] = buy_url
-    ndoc[:spider_id] = doc["spider_id"] if doc["spider_id"]
-    ndoc[:origin_platform] = doc["origin_platform"] if doc["origin_platform"]
-    ndoc[:pno] = pno
-
-    (1..4).each do |i|
-      if doc["desc#{i}"] and doc["desc#{i}"] != ""
-        ndoc[:desc] += doc["desc#{i}"]
-      end
-    end
-
-    ndoc[:category_id] = 3
-    ndoc[:category_id] = doc["category_id"] if doc["category_id"]
-    ndoc[:sub_category_id] = doc["sub_cat_id"] if doc["sub_cat_id"] && doc["sub_cat_id"].to_i > 0
-
-    ndoc[:imgs] = []
-    ndoc[:dp_imgs] = []
-
-    if doc["dp_imgs"] and doc["dp_imgs"] != ""
-      ndoc[:dp_imgs]  = doc["dp_imgs"]
-    end
-    if doc["simg"] and doc["simg"] != ""
-      doc["imgs"] += doc["simg"]
-    end
-    if doc["imgs"]
-      doc["imgs"] = doc["imgs"].uniq
-      doc["imgs"].each do |img|
-        if img =~ /gif$/ or img == "" or img == nil
-          next
-        end
-
-        ndoc[:imgs] << img
-      end
-    end
-    ndoc
-end
-
 
 def process(doc, update=false)
 end
@@ -131,10 +49,10 @@ namespace :matter do
      spiders = Brand.find(brand_id).spiders
      spiders.each do |spider|
       spider_id = spider.id
-      cats = Category.where("id >= 3").where("id < 15")
-       cats.each do |c|
-         cat_id = c.id
-         pkey = "brand_#{brand_id}_spider_#{spider_id}_product_pages_#{cat_id}"
+      spider_pages = spider.get_spider_pages
+      spider_pages.each do |c|
+         page_id = c.id
+         pkey = "brand_#{brand_id}_spider_#{spider_id}_spider_pages_#{page_id}"
          lks = @redis.lrange(pkey, 0, -1)
          brand_cat_list_length = lks.length
          p pkey,lks.length
@@ -146,20 +64,20 @@ namespace :matter do
            begin
              p link
              next if link.nil?
-             if cat_id==9 || cat_id==10
-               link_sub_cat_id = link.split(",")
-               if link_sub_cat_id.length == 2
-                 link = link_sub_cat_id.last
-               else
-                 link = link_sub_cat_id.first
-               end
-             end
              link_docid = make_docid(link)
              doc_yml = @redis.get link_docid
              next if doc_yml.nil?
+
              doc = YAML::load( doc_yml )
              doc['docid'] = link_docid
              doc['link'] = link
+             doc['category_id'] = c.parent_id
+             u = User.create_by_brand(brand_id)
+             doc['user_id'] = u.id
+             doc['page_id'] = page_id
+             doc['sub_category_id'] = c.get_category(u.id).id
+             #========
+
              p doc
              Matter.create_from_hash(doc)
              #process( make_doc(doc) )