Skip to content

Commit

Permalink
rss: remove 3dm news content
Browse files Browse the repository at this point in the history
  • Loading branch information
DIYgod committed Jun 21, 2018
1 parent 37d2d3c commit 3587861
Showing 1 changed file with 1 addition and 66 deletions.
67 changes: 1 addition & 66 deletions routes/3dm/news.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,81 +21,16 @@ module.exports = async (ctx) => {
const list = $('.dowlnewslist a');
const items = [];

for (let i = 0; i < Math.min(list.length, 10); i++) {
for (let i = 0; i < list.length; i++) {
let item = $(list[i]);
const url = item.attr('href');

let content = '';
const urlBase = url.replace(/.html/, '');
let total = 1;

const value = await ctx.cache.get(url);
if (value) {
item = JSON.parse(value);
} else {
// 抓取分页
for (let j = 1; ; j++) {
let response;

const u = j === 1 ? url : urlBase + '_' + j + '.html';

try {
response = await axios({
method: 'get',
url: u,
headers: {
'User-Agent': config.ua,
},
});
} catch (e) {
break;
}

const page = cheerio.load(response.data, {
decodeEntities: false,
});

if (type === 'download') {
content = page('.jieshao').html();
} else {
// 提取页数
if (j === 1) {
if (page('.pagelistbox').length === 0) {
total = 1;
} else {
total = parseInt(
page('.pagelistbox')
.find('span')
.html()
.match(/ (\S*) /)[1]
);
}
}

// 去除不需要的元素
page('.page_fenye').remove(); // 翻页
page('.con p')
.last()
.remove(); // 专题跳转
if (total > 1) {
page('.con p')
.last()
.remove(); // 快速翻页提示
}

content += page('.con div')
.next()
.html();
}

if (j >= total) {
break;
}
}

item = {
title: item.find('p').text(),
description: content,
pubDate: item.find('span').text(),
link: url,
guid: url,
Expand Down

0 comments on commit 3587861

Please sign in to comment.