Skip to content

Commit

Permalink
fix(route): /yomiuri/news fetching error due to frame changed (DIYgod…
Browse files Browse the repository at this point in the history 
…#7809)
  • Loading branch information
@Arracc
Arracc authored Jul 17, 2021
1 parent b008df1 commit ec40eaa
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 14 deletions.
8 changes: 4 additions & 4 deletions docs/traditional-media.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,11 +531,11 @@ IT・科学 tech_science

<Route author="Arracc" example="/yomiuri/news" path="/yomiuri/:category" :paramsDesc="['板块']">

无料全文，综合页文章标题添加板块标签。
无料全文，综合页 (新着・速報) 文章标题补充板块标签。

| 総合 | 社会 | 政治 | 経済 | スポーツ | 国際 | 科学・ＩＴ | 選挙・世論調査 | エンタメ・文化 | 囲碁・将棋 | ライフ | 地域 | 社説 |
| ---- | -------- | -------- | ------- | -------- | ----- | ---------- | -------------- | -------------- | ---------- | ------ | ----- | --------- |
| news | national | politics | economy | sports | world | science | election | culture | igoshougi | life | local | editorial |
| 新着・速報 | 　　社会 | 政治 | 経済 | スポーツ | 国際 | 科学・ＩＴ | 選挙・世論調査 | エンタメ・文化 | 囲碁・将棋 | ライフ | 地域 | 社説 |
| ---------- | -------- | -------- | ------- | -------- | ----- | ---------- | -------------- | -------------- | ---------- | ------ | ----- | --------- |
| 　news 　 | national | politics | economy | sports | world | science | election | culture | igoshougi | life | local | editorial |

</Route>

Expand Down
93 changes: 83 additions & 10 deletions lib/routes/yomiuri/news.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const timezone = require('@/utils/timezone');

module.exports = async (ctx) => {
const category = ctx.params.category;
Expand All @@ -12,30 +13,99 @@ module.exports = async (ctx) => {
const data = response.data;
const $ = cheerio.load(data);

let category_name = '';
let categoryName;
let list = null;
if (category === 'news') {
category_name = '総合';
list = $('ul.news-top-upper-content-topics-content-list.p-list').children('li').not('.p-member-only');
categoryName = '新着・速報';
const urlPrefix = 'https://www.yomiuri.co.jp/y_ajax/';

// get list1
const param1 = $('input#latest_list_news_other_params').val();
const response1 = await got({
method: 'get',
url: urlPrefix + 'latest_list_news/' + param1 + '/?action=latest_list_news&others=' + param1,
});
const data1 = response1.data;
const firstArticleList = cheerio.load(data1);

// get list2
const param2 = $('input#latest_list_news_other_params2').val();
const response2 = await got({
method: 'get',
url: urlPrefix + 'latest_list_news2/' + param2 + '/?action=latest_list_news2&others=' + param2,
});
const data2 = response2.data;
const secondArticleList = cheerio.load(data2);
list = secondArticleList('article').add(firstArticleList('article'));

list = list.map((index, item) => {
item = $(item);
// remove matome title
const matomeTitle = item.find('div.c-matome-title');
if (matomeTitle.length !== 0) {
matomeTitle.remove();
}
// remove member only
const lockedIcon = item.find('svg[class=icon-locked]');
if (lockedIcon.length === 0) {
return item;
} else {
return null;
}
});
} else {
category_name = $('h1.p-header-category-current-title').text();
list = $('div.p-category-organization,.p-category-time-series').find('li').not('.p-member-only').not('.p-ad-list-item');
let parentSelector = 'div.p-category-organization,.p-category-time-series';
// igoshougi channel sepcial div selector
if (category === 'igoshougi') {
parentSelector = 'div.uni-news-igoshougi-top';
}
categoryName = $('h1.p-header-category-current-title,h1.p-header-category-content-title').text();
// p-header-category-content-title
list = $(parentSelector).find('li').not('.p-member-only').not('.p-ad-list-item');
list = list.map((index, item) => {
item = $(item);
// remove matome title
const matomeTitle = item.find('div.c-matome-title');
if (matomeTitle.length !== 0) {
matomeTitle.remove();
}
return item;
});
}

// format category name
if (categoryName !== '') {
categoryName = ' - ' + categoryName.trim();
}

const items = await Promise.all(
list
.map(async (index, item) => {
item = $(item);
const link = item.find('a').attr('href');

let link = '';
if (categoryName === '新着・速報') {
link = item.find('h3').children('a').attr('href');
} else {
link = item.find('a').attr('href');
}

const description = await ctx.cache.tryGet(link, async () => {
const response = await got.get(link);
const $ = cheerio.load(response.data);
return $('div.p-main-contents').html();
const mainContent = $('div.p-main-contents');
// remove articles recomended
const recommendNode = mainContent.find('aside.ev-article-manual-related__article-inline');
if (recommendNode.length !== 0) {
recommendNode.remove();
}
return mainContent.html();
});

let tag = '';
if (category_name === '総合') {
if (categoryName === '新着・速報') {
const tag_table = {
news: '総合',
news: '新着・速報',
national: '社会',
politics: '政治',
economy: '経済',
Expand All @@ -49,22 +119,25 @@ module.exports = async (ctx) => {
local: '地域',
editorial: '社説',
medical: '医療・健康',
olympic: 'オリンピック',
};
const tag_value = tag_table[link.split('/')[3]];
tag = tag_value ? tag_value : 'その他';
tag = '[' + tag + '] ';
}

return {
title: tag + item.find('a').text(),
description,
pubDate: timezone(new Date(item.find('time').prop('datetime')), +9),
link,
};
})
.get()
);

ctx.state.data = {
title: '読売新聞-' + category_name,
title: '読売新聞' + categoryName,
link: url,
item: items,
};
Expand Down

0 comments on commit ec40eaa

Please sign in to comment.