Skip to content

Commit

Permalink
redis可以进行库切换
Browse files Browse the repository at this point in the history
  • Loading branch information
gudegg committed Dec 11, 2016
1 parent 1ee2013 commit ff894a1
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 7 deletions.
2 changes: 2 additions & 0 deletions config.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ url=(127.0.0.1:3306)/baidu
address=127.0.0.1:6379
#密码不存在直接留空 如:password=
password=123456
#库选择,redis默认使用0号库
database=0
#设置爬取数据模式 直接从某个uk开始爬取的话设置mode=1且必须设置开始爬取的uk值,如果数据库avaiuk表存储了热门uk的话 设置mode=2且uk无需设置
[Mode]
mode=1
Expand Down
23 changes: 16 additions & 7 deletions spider.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import (

var db *sql.DB
var err error
var username, password, url, address, redis_Pwd, mode, logLevel string
var username, password, url, address, redis_Pwd, mode, logLevel, redis_db string
var redis_Database int
var ConfError error
var cfg *goconfig.ConfigFile
//Mysql Redis初始化
Expand Down Expand Up @@ -57,6 +58,14 @@ func init() {
if ConfError != nil {
panic("读取Redis password错误")
}
redis_db, ConfError = cfg.GetValue("Redis", "database")
if ConfError != nil {
redis_db = "0"
}
redis_Database, ConfError = strconv.Atoi(redis_db)
if ConfError != nil {
redis_Database = 0
}
var dataSourceName bytes.Buffer
dataSourceName.WriteString(username)
dataSourceName.WriteString(":")
Expand Down Expand Up @@ -114,7 +123,7 @@ func initRedisPool() {
return nil, cErr
}
} else {
conn, cErr = redis.Dial("tcp", address, redis.DialPassword(redis_Pwd))
conn, cErr = redis.Dial("tcp", address, redis.DialPassword(redis_Pwd), redis.DialDatabase(redis_Database))
if cErr != nil {
log.Errorf("Redis初始化失败,请检查配置是否填写正确,key存储切换到文件模式")
return nil, cErr
Expand Down Expand Up @@ -201,7 +210,7 @@ func main() {

}
}
log.Info("已经递归爬取完成,请切换新的热门uk或者存储到新的热门uk到数据库表avaiuk中")
log.Info("已经递归爬取完成,请切换新的热门uk或者存储新的热门uk到数据库表avaiuk中")
time.Sleep(time.Second * 2)

}
Expand Down Expand Up @@ -444,10 +453,10 @@ func IndexResource(uk int64) {

for _, v := range yundata.Feedata.Records {
if strings.Compare(v.Feed_type, "share") == 0 {
db.Exec("insert into sharedata(title,shareid,uinfo_id,category) values(?,?,?,?)", v.Title, v.Shareid, uinfoId,v.Category)
db.Exec("insert into sharedata(title,shareid,uinfo_id,category) values(?,?,?,?)", v.Title, v.Shareid, uinfoId, v.Category)
log.Info("insert share")
} else if strings.Compare(v.Feed_type, "album") == 0 {
db.Exec("insert into sharedata(title,album_id,uinfo_id,category) values(?,?,?,?)", v.Title, v.Album_id, uinfoId,v.Category)
db.Exec("insert into sharedata(title,album_id,uinfo_id,category) values(?,?,?,?)", v.Title, v.Album_id, uinfoId, v.Category)
log.Info("insert album")
}

Expand All @@ -464,10 +473,10 @@ func IndexResource(uk int64) {
if yundata != nil {
for _, v := range yundata.Feedata.Records {
if strings.Compare(v.Feed_type, "share") == 0 {
db.Exec("insert into sharedata(title,shareid,uinfo_id,category) values(?,?,?,?)", v.Title, v.Shareid, uinfoId,v.Category)
db.Exec("insert into sharedata(title,shareid,uinfo_id,category) values(?,?,?,?)", v.Title, v.Shareid, uinfoId, v.Category)
log.Info("insert share")
} else if strings.Compare(v.Feed_type, "album") == 0 {
db.Exec("insert into sharedata(title,album_id,uinfo_id,category) values(?,?,?,?)", v.Title, v.Album_id, uinfoId,v.Category)
db.Exec("insert into sharedata(title,album_id,uinfo_id,category) values(?,?,?,?)", v.Title, v.Album_id, uinfoId, v.Category)
log.Info("insert album")
}
}
Expand Down

0 comments on commit ff894a1

Please sign in to comment.