-
Notifications
You must be signed in to change notification settings - Fork 25
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
王林
committed
Apr 21, 2019
1 parent
4d5ec89
commit a9f30e6
Showing
21 changed files
with
2,328 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
node_modules | ||
!.gitkeep |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
## 豆瓣API服务 | ||
通过phantom无头浏览器爬取豆瓣页面,分析页面结构来解析并输出json类型数据 | ||
|
||
## 缘起 | ||
豆瓣之前有提供官方开放API,虽然每天调用接口有上限,但是至少可以使用,后来不知为何突然关闭了且不允许个人申请。笔者有部分个人项目有使用豆瓣的接口,所以突然关闭给笔者带来了不小的困扰,所以就有了这个项目 | ||
|
||
## 代码目录结构 | ||
- `config`: 项目的配置信息 | ||
- `middlewares`: 中间件函数 | ||
- `controllers`: 将路由转发到对应的model上 | ||
- `models`: 业务逻辑 | ||
- `routes`: 路由 | ||
- `utils`: 公共工具函数 | ||
- `app.js`: 服务入口 | ||
|
||
## 路由区分 | ||
- /book 书籍部分接口 | ||
- /movie 影视部分接口 | ||
- /music 音乐部分接口 | ||
|
||
## 接口详解 | ||
### 公共 | ||
|
||
### 书籍 | ||
- /book/list?key=&page= 根据关键词搜索书籍列表,只有概要,实际可能作用不大。豆瓣分页比较迷,如果页数较大的情况下是不会透出所有页数的,所以该接口统一不透出总页数数据,所以当请求某一页没有数据时那之前的一页应该就是最后一页。√ | ||
- /book/detail?url= 获取某本书籍的详细信息,url来自list或list_pro接口 √ | ||
|
||
### 影视 | ||
- /movie/list?key=&page= 根据关键词搜索影视列表,同书籍对应接口 √ | ||
- /movie/detail?url= 获取影视的详细信息,同书籍对应接口 √ | ||
|
||
## 特别注意 | ||
- 因为是通过爬取页面来抓取数据,所以会有以下问题: | ||
- --数据较慢,且有一定失败概率 | ||
- --因为是通过分析页面结构来输出数据,页面结构很容易变动,所以相当不稳定,所以使用场景慎重考虑 | ||
|
||
## 部署 | ||
- 机器需要安装phantom |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
//引入框架 | ||
const express = require('express'); | ||
|
||
//配置文件 | ||
const config = require('./config/config'); | ||
|
||
//工具函数 | ||
const { getIPAdress, log } = require('./utils/utils'); | ||
|
||
//跨域处理 | ||
const cors = require('./middlewares/cors'); | ||
|
||
//引入路由 | ||
const router = require('./routes/index'); | ||
|
||
//创建应用 | ||
const app = express(); | ||
|
||
//跨域处理 | ||
app.all('*', (req, res, next) => { | ||
log(req.method + ' ' + req.originalUrl); | ||
next(); | ||
} , cors); | ||
|
||
//路由 | ||
app.use(router); | ||
|
||
//处理404响应 | ||
app.use(function (req, res, next) { | ||
log(404); | ||
}); | ||
|
||
//错误处理 | ||
app.use(function (err, req, res, next) { | ||
log('兜底错误处理:' + err); | ||
}); | ||
|
||
//监听端口 | ||
app.listen(config.port, () => { | ||
let ip = getIPAdress(); | ||
log('服务成功启动'); | ||
log(`访问地址:http://localhost:${config.port}`); | ||
log(`访问地址:http://${ip}:${config.port}`); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
module.exports = { | ||
//允许跨域的白名单 | ||
corsWhiteList: [], | ||
|
||
//服务运行的端口 | ||
port: 8085 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
const bookModel = require('../models/book'); | ||
|
||
const book = { | ||
//根据关键词搜索书籍列表 | ||
'GET list': (req, res) => { | ||
bookModel.list(req, res); | ||
}, | ||
|
||
//获取某本书籍的详细信息 | ||
'GET detail': (req, res) => { | ||
bookModel.detail(req, res); | ||
} | ||
} | ||
|
||
module.exports = book; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
const movieModel = require('../models/movie'); | ||
|
||
const movie = { | ||
//根据关键词搜索影视列表 | ||
'GET list': (req, res) => { | ||
movieModel.list(req, res); | ||
}, | ||
|
||
//获取影视的详细信息 | ||
'GET detail': (req, res) => { | ||
movieModel.detail(req, res); | ||
} | ||
} | ||
|
||
module.exports = movie; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
const musicModel = require('../models/music'); | ||
|
||
const music = { | ||
//根据关键词搜索音乐列表 | ||
'GET list': (req, res) => { | ||
musicModel.list(req, res); | ||
}, | ||
|
||
//获取音乐的详细信息 | ||
'GET detail': (req, res) => { | ||
musicModel.detail(req, res); | ||
} | ||
} | ||
|
||
module.exports = music; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
//跨域设置、白名单处理 | ||
const config = require('../config/config'); | ||
const whiteList = config.corsWhiteList; | ||
const { log } = require('../utils/utils'); | ||
|
||
module.exports = function (req, res, next) { | ||
if (req.headers && req.headers.origin && whiteList.indexOf(req.headers.origin) !== -1) { | ||
log('允许跨域:' + req.headers.origin); | ||
res.setHeader('Access-Control-Allow-Origin', req.headers.origin); | ||
res.setHeader('Access-Control-Allow-Credentials', true); | ||
res.setHeader('Access-Control-Allow-Methods', 'GET,POST,OPTIONS,PUT,DELETE'); | ||
res.setHeader("Access-Control-Allow-Headers", "Content-Type,XFILENAME,XFILECATEGORY,XFILESIZE") | ||
} | ||
next(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
const { log } = require('../utils/utils'); | ||
const listModel = require('./list'); | ||
const detailModel = require('./detail'); | ||
|
||
/** | ||
* 书籍类 | ||
* | ||
* @class Book | ||
*/ | ||
class Book { | ||
/** | ||
* 构造函数 | ||
* @memberof Book | ||
*/ | ||
constructor() { | ||
//豆瓣书籍列表页URL | ||
this.bookPageUrl = 'https://book.douban.com/subject_search'; | ||
//搜索类目 | ||
this.catNum = '1001'; | ||
//每页数量 | ||
this.pageSize = 15;//豆瓣一页默认为15 | ||
} | ||
|
||
/** | ||
* 根据关键词搜索书籍列表 | ||
* | ||
* @memberof Book | ||
*/ | ||
async list(req, res) { | ||
log('根据关键词搜索书籍列表'); | ||
await listModel.list(req, res, this.bookPageUrl, this.catNum, this.pageSize); | ||
} | ||
|
||
/** | ||
* 获取某本书籍的详细信息 | ||
* | ||
* @param {*} req | ||
* @param {*} res | ||
* @memberof Book | ||
*/ | ||
async detail(req, res) { | ||
log('获取某本书籍的详细信息'); | ||
await detailModel.detail(req, res, this.catNum); | ||
} | ||
} | ||
|
||
module.exports = new Book(); |
Oops, something went wrong.