Skip to content

Commit

Permalink
fixing except block directly handling baseException
Browse files Browse the repository at this point in the history
  • Loading branch information
@slowy07
slowy07 committed Jul 17, 2021
1 parent 4ace637 commit b97e0a6
Showing 1 changed file with 132 additions and 111 deletions.
243 changes: 132 additions & 111 deletions news_articles__scraper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,61 +11,64 @@

# ! pip install newspaper3k

# importing necessary libraries
from bs4 import BeautifulSoup
import requests
import urllib
import pandas as pd
from newspaper import Article
import pickle
import re
import sys
import urllib

import pandas as pd
import requests

# importing necessary libraries
from bs4 import BeautifulSoup
from newspaper import Article

# Extracting links for all the pages (1 to 158) of boomlive fake news section
fakearticle_links = []
for i in range(1, 159):
url = 'https://www.boomlive.in/fake-news/' + str(i)
try:
# this might throw an exception if something goes wrong.
page=requests.get(url)

# send requests
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')

# Collecting all the links in a list
for content in soup.find_all('h2', attrs={'class':'entry-title'}):
link = content.find('a')
fakearticle_links.append(link.get('href'))
# this describes what to do if an exception is thrown
except Exception as e:
# get the exception information
error_type, error_obj, error_info = sys.exc_info()
#print the link that cause the problem
print ('ERROR FOR LINK:',url)
#print error info and line that threw the exception
print (error_type, 'Line:', error_info.tb_lineno)
continue
url = "https://www.boomlive.in/fake-news/" + str(i)
try:
# this might throw an exception if something goes wrong.
page = requests.get(url)

# send requests
page = requests.get(url)
soup = BeautifulSoup(page.text, "html.parser")

# Collecting all the links in a list
for content in soup.find_all("h2", attrs={"class": "entry-title"}):
link = content.find("a")
fakearticle_links.append(link.get("href"))

# this describes what to do if an exception is thrown
except Exception as e:
# get the exception information
error_type, error_obj, error_info = sys.exc_info()
# print the link that cause the problem
print("ERROR FOR LINK:", url)
# print error info and line that threw the exception
print(error_type, "Line:", error_info.tb_lineno)
continue

fakearticle_links[:5]

len(fakearticle_links)

fakearticle_links[1888:]

import matplotlib.pyplot as plt
import pandas as pd

import numpy as np
import matplotlib.pyplot as plt

"""We have to modify the links so that the links actually work as we can see that the string extracted is the last part of the url!
**We have to add 'https://www.boomlive.in/fake-news' to the extracted links.**
"""

# Modify the links so that it takes us to the particular website
str1 = 'https://www.boomlive.in/fake-news'
fakearticle_links = [str1+lnk for lnk in fakearticle_links]
str1 = "https://www.boomlive.in/fake-news"
fakearticle_links = [str1 + lnk for lnk in fakearticle_links]

fakearticle_links[6:10]

Expand All @@ -75,30 +78,37 @@
"""

# Create a dataset for storing the news articles
news_dataset = pd.DataFrame(fakearticle_links, columns=['URL'])
news_dataset = pd.DataFrame(fakearticle_links, columns=["URL"])

news_dataset.head()

title, text, summary, keywords, published_on, author = [], [], [], [], [], [] # Creating empty lists to store the data
title, text, summary, keywords, published_on, author = (
[],
[],
[],
[],
[],
[],
) # Creating empty lists to store the data
for Url in fakearticle_links:
article = Article(Url)

#Call the download and parse methods to download information
try:
article.download()
article.parse()
article.nlp()
except Exception as error:
print(f"exception : {error}")
pass
# Scrape the contents of article
title.append(article.title) # extracts the title of the article
text.append(article.text) # extracts the whole text of article
summary.append(article.summary) # gives us a summary abou the article
keywords.append(', '.join(article.keywords)) # the main keywords used in it
published_on.append(article.publish_date) # the date on which it was published
author.append(article.authors) # the authors of the article
article = Article(Url)

# Call the download and parse methods to download information
try:
article.download()
article.parse()
article.nlp()
except Exception as error:
print(f"exception : {error}")
pass

# Scrape the contents of article
title.append(article.title) # extracts the title of the article
text.append(article.text) # extracts the whole text of article
summary.append(article.summary) # gives us a summary abou the article
keywords.append(", ".join(article.keywords)) # the main keywords used in it
published_on.append(article.publish_date) # the date on which it was published
author.append(article.authors) # the authors of the article

"""**Checking the lists created**"""

Expand All @@ -111,107 +121,118 @@
author[6]

# Adding the columns in the fake news dataset
news_dataset['title'] = title
news_dataset['text'] = text
news_dataset['keywords'] = keywords
news_dataset['published date'] = published_on
news_dataset['author'] = author
news_dataset["title"] = title
news_dataset["text"] = text
news_dataset["keywords"] = keywords
news_dataset["published date"] = published_on
news_dataset["author"] = author

# Check the first five columns of dataset created
news_dataset.head()

"""**Converting the dataset to a csv file**"""

news_dataset.to_csv('Fake_news.csv')
news_dataset.to_csv("Fake_news.csv")

"""**Reading the csv file**"""

df = pd.read_csv('Fake_news.csv')
df = pd.read_csv("Fake_news.csv")

# Checking the last 5 rows of the csv file
df.tail(5)

"""**Download the csv file in local machine**"""

from google.colab import files
files.download('Fake_news.csv')

files.download("Fake_news.csv")

"""**Scraping news from Times of India**"""

TOIarticle_links = [] # Creating an empty list of all the urls of news from Times of India site
TOIarticle_links = (
[]
) # Creating an empty list of all the urls of news from Times of India site

# Extracting links for all the pages (2 to 125) of boomlive fake news section
for i in range(2, 126):
url = 'https://timesofindia.indiatimes.com/news/' + str(i)
try:
# send requests
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')

# Collecting all the links in a list
for content in soup.find_all('span', attrs={'class':'w_tle'}):
link = content.find('a')
TOIarticle_links.append(link.get('href'))
# this describes what to do if an exception is thrown
except Exception as e:
# get the exception information
error_type, error_obj, error_info = sys.exc_info()
#print the link that cause the problem
print ('ERROR FOR LINK:',url)
#print error info and line that threw the exception
print (error_type, 'Line:', error_info.tb_lineno)
continue
url = "https://timesofindia.indiatimes.com/news/" + str(i)

try:
# send requests
page = requests.get(url)
soup = BeautifulSoup(page.text, "html.parser")

# Collecting all the links in a list
for content in soup.find_all("span", attrs={"class": "w_tle"}):
link = content.find("a")
TOIarticle_links.append(link.get("href"))

# this describes what to do if an exception is thrown
except Exception as e:
# get the exception information
error_type, error_obj, error_info = sys.exc_info()
# print the link that cause the problem
print("ERROR FOR LINK:", url)
# print error info and line that threw the exception
print(error_type, "Line:", error_info.tb_lineno)
continue

TOIarticle_links[6:15]

len(TOIarticle_links)

str2 = 'https://timesofindia.indiatimes.com'
TOIarticle_links = [str2+lnk for lnk in TOIarticle_links if lnk[0]=='/']
str2 = "https://timesofindia.indiatimes.com"
TOIarticle_links = [str2 + lnk for lnk in TOIarticle_links if lnk[0] == "/"]

TOIarticle_links[5:8]

len(TOIarticle_links)

title, text, summary, keywords, published_on, author = [], [], [], [], [], [] # Creating empty lists to store the data
title, text, summary, keywords, published_on, author = (
[],
[],
[],
[],
[],
[],
) # Creating empty lists to store the data
for Url in TOIarticle_links:
article = Article(Url)

#Call the download and parse methods to download information
try:
article.download()
article.parse()
article.nlp()
except:
pass
# Scrape the contents of article
title.append(article.title) # extracts the title of the article
text.append(article.text) # extracts the whole text of article
summary.append(article.summary) # gives us a summary abou the article
keywords.append(', '.join(article.keywords)) # the main keywords used in it
published_on.append(article.publish_date) # the date on which it was published
author.append(article.authors) # the authors of the article
article = Article(Url)

# Call the download and parse methods to download information
try:
article.download()
article.parse()
article.nlp()
except Exception:
pass

# Scrape the contents of article
title.append(article.title) # extracts the title of the article
text.append(article.text) # extracts the whole text of article
summary.append(article.summary) # gives us a summary abou the article
keywords.append(", ".join(article.keywords)) # the main keywords used in it
published_on.append(article.publish_date) # the date on which it was published
author.append(article.authors) # the authors of the article

title[5]

TOI_dataset = pd.DataFrame(TOIarticle_links, columns=['URL'])
TOI_dataset = pd.DataFrame(TOIarticle_links, columns=["URL"])
# Adding the columns in the TOI news dataset
TOI_dataset['title'] = title
TOI_dataset['text'] = text
TOI_dataset['keywords'] = keywords
TOI_dataset['published date'] = published_on
TOI_dataset['author'] = author
TOI_dataset["title"] = title
TOI_dataset["text"] = text
TOI_dataset["keywords"] = keywords
TOI_dataset["published date"] = published_on
TOI_dataset["author"] = author

TOI_dataset.head()

TOI_dataset.to_csv('TOI_news_dataset.csv')
TOI_dataset.to_csv("TOI_news_dataset.csv")

dt = pd.read_csv('TOI_news_dataset.csv')
dt = pd.read_csv("TOI_news_dataset.csv")

dt.tail(3)

from google.colab import files
files.download('TOI_news_dataset.csv')

files.download("TOI_news_dataset.csv")

0 comments on commit b97e0a6

Please sign in to comment.