Skip to content

Commit

Permalink
基本完成
Browse files Browse the repository at this point in the history
  • Loading branch information
王林 committed Apr 21, 2019
1 parent 4d5ec89 commit a9f30e6
Show file tree
Hide file tree
Showing 21 changed files with 2,328 additions and 0 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
node_modules
!.gitkeep
38 changes: 38 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
## 豆瓣API服务
通过phantom无头浏览器爬取豆瓣页面,分析页面结构来解析并输出json类型数据

## 缘起
豆瓣之前有提供官方开放API,虽然每天调用接口有上限,但是至少可以使用,后来不知为何突然关闭了且不允许个人申请。笔者有部分个人项目有使用豆瓣的接口,所以突然关闭给笔者带来了不小的困扰,所以就有了这个项目

## 代码目录结构
- `config`: 项目的配置信息
- `middlewares`: 中间件函数
- `controllers`: 将路由转发到对应的model上
- `models`: 业务逻辑
- `routes`: 路由
- `utils`: 公共工具函数
- `app.js`: 服务入口

## 路由区分
- /book 书籍部分接口
- /movie 影视部分接口
- /music 音乐部分接口

## 接口详解
### 公共

### 书籍
- /book/list?key=&page= 根据关键词搜索书籍列表,只有概要,实际可能作用不大。豆瓣分页比较迷,如果页数较大的情况下是不会透出所有页数的,所以该接口统一不透出总页数数据,所以当请求某一页没有数据时那之前的一页应该就是最后一页。√
- /book/detail?url= 获取某本书籍的详细信息,url来自list或list_pro接口 √

### 影视
- /movie/list?key=&page= 根据关键词搜索影视列表,同书籍对应接口 √
- /movie/detail?url= 获取影视的详细信息,同书籍对应接口 √

## 特别注意
- 因为是通过爬取页面来抓取数据,所以会有以下问题:
- --数据较慢,且有一定失败概率
- --因为是通过分析页面结构来输出数据,页面结构很容易变动,所以相当不稳定,所以使用场景慎重考虑

## 部署
- 机器需要安装phantom
44 changes: 44 additions & 0 deletions app.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
//引入框架
const express = require('express');

//配置文件
const config = require('./config/config');

//工具函数
const { getIPAdress, log } = require('./utils/utils');

//跨域处理
const cors = require('./middlewares/cors');

//引入路由
const router = require('./routes/index');

//创建应用
const app = express();

//跨域处理
app.all('*', (req, res, next) => {
log(req.method + ' ' + req.originalUrl);
next();
} , cors);

//路由
app.use(router);

//处理404响应
app.use(function (req, res, next) {
log(404);
});

//错误处理
app.use(function (err, req, res, next) {
log('兜底错误处理:' + err);
});

//监听端口
app.listen(config.port, () => {
let ip = getIPAdress();
log('服务成功启动');
log(`访问地址:http://localhost:${config.port}`);
log(`访问地址:http://${ip}:${config.port}`);
});
7 changes: 7 additions & 0 deletions config/config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
module.exports = {
//允许跨域的白名单
corsWhiteList: [],

//服务运行的端口
port: 8085
}
15 changes: 15 additions & 0 deletions controllers/book.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
const bookModel = require('../models/book');

const book = {
//根据关键词搜索书籍列表
'GET list': (req, res) => {
bookModel.list(req, res);
},

//获取某本书籍的详细信息
'GET detail': (req, res) => {
bookModel.detail(req, res);
}
}

module.exports = book;
15 changes: 15 additions & 0 deletions controllers/movie.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
const movieModel = require('../models/movie');

const movie = {
//根据关键词搜索影视列表
'GET list': (req, res) => {
movieModel.list(req, res);
},

//获取影视的详细信息
'GET detail': (req, res) => {
movieModel.detail(req, res);
}
}

module.exports = movie;
15 changes: 15 additions & 0 deletions controllers/music.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
const musicModel = require('../models/music');

const music = {
//根据关键词搜索音乐列表
'GET list': (req, res) => {
musicModel.list(req, res);
},

//获取音乐的详细信息
'GET detail': (req, res) => {
musicModel.detail(req, res);
}
}

module.exports = music;
15 changes: 15 additions & 0 deletions middlewares/cors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//跨域设置、白名单处理
const config = require('../config/config');
const whiteList = config.corsWhiteList;
const { log } = require('../utils/utils');

module.exports = function (req, res, next) {
if (req.headers && req.headers.origin && whiteList.indexOf(req.headers.origin) !== -1) {
log('允许跨域:' + req.headers.origin);
res.setHeader('Access-Control-Allow-Origin', req.headers.origin);
res.setHeader('Access-Control-Allow-Credentials', true);
res.setHeader('Access-Control-Allow-Methods', 'GET,POST,OPTIONS,PUT,DELETE');
res.setHeader("Access-Control-Allow-Headers", "Content-Type,XFILENAME,XFILECATEGORY,XFILESIZE")
}
next();
}
47 changes: 47 additions & 0 deletions models/book.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
const { log } = require('../utils/utils');
const listModel = require('./list');
const detailModel = require('./detail');

/**
* 书籍类
*
* @class Book
*/
class Book {
/**
* 构造函数
* @memberof Book
*/
constructor() {
//豆瓣书籍列表页URL
this.bookPageUrl = 'https://book.douban.com/subject_search';
//搜索类目
this.catNum = '1001';
//每页数量
this.pageSize = 15;//豆瓣一页默认为15
}

/**
* 根据关键词搜索书籍列表
*
* @memberof Book
*/
async list(req, res) {
log('根据关键词搜索书籍列表');
await listModel.list(req, res, this.bookPageUrl, this.catNum, this.pageSize);
}

/**
* 获取某本书籍的详细信息
*
* @param {*} req
* @param {*} res
* @memberof Book
*/
async detail(req, res) {
log('获取某本书籍的详细信息');
await detailModel.detail(req, res, this.catNum);
}
}

module.exports = new Book();
Loading

0 comments on commit a9f30e6

Please sign in to comment.