Skip to content

Commit

Permalink
add flake8 and format code
Browse files Browse the repository at this point in the history
  • Loading branch information
@jachinlin
jachinlin committed Aug 30, 2019
1 parent 4ba807b commit d697495
Show file tree
Hide file tree
Showing 22 changed files with 293 additions and 167 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/geektime-dl-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ jobs:
pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
flake8 .
- name: Test with pytest
env:
account: ${{ secrets.account }}
password: ${{ secrets.password }}
run: |
pip install pytest
python -m pytest
pip install pytest codecov pytest-cov
python -m pytest --cov=./
codecov
4 changes: 2 additions & 2 deletions geektime_dl/cli/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# coding=utf8

from .command import Command, main, save_cfg
from . import login, query, mp4, mp3, ebook
from geektime_dl.cli.command import Command, main, save_cfg # noqa: F401
from geektime_dl.cli import login, query, mp4, mp3, ebook # noqa: F401
81 changes: 47 additions & 34 deletions geektime_dl/cli/command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@
import configparser
import argparse

from ..utils._logging import logger

from geektime_dl.utils.log import logger

commands = {}


geektime_cfg = os.path.abspath('./geektime.cfg')
cwd = os.path.abspath('.')

Expand All @@ -34,11 +32,16 @@ def work(self: 'Command', args: list):
return self.run(cfg)


Command = CommandType('Command', (object,), {'run': lambda self, args: None, 'work': work})
Command = CommandType(
'Command',
(object,),
{'run': lambda self, args: None, 'work': work}
)


class Help(Command):
"""Display the list of available commands"""

def run(self, cfg):
result = ["Available commands:"]
names = list(commands)
Expand All @@ -47,15 +50,16 @@ def run(self, cfg):
name = k.ljust(padding, ' ')
doc = (commands[k].__doc__ or '').split('\n')[0]
result.append(" %s%s" % (name, doc))
result.append("\nUse '%s <command> --help' for individual command help." % sys.argv[0].split(os.path.sep)[-1])
result.append(
"\nUse '{} <command> --help' for individual command help.".format(
sys.argv[0].split(os.path.sep)[-1]))

result = '\n'.join(result)
sys.stdout.write(result)
return result


def _load_cfg(cfg_file: str) -> dict:

p = configparser.RawConfigParser()
cfg = dict()
try:
Expand Down Expand Up @@ -95,37 +99,46 @@ def save_cfg(cfg: dict) -> None:

def parse_config(args: list) -> dict:
parser = argparse.ArgumentParser(prog='{} <command>'.format(sys.argv[0]))
parser.add_argument("--config", dest="config", type=str, default=geektime_cfg,
help="specify alternate config file")
parser.add_argument("--config", dest="config", type=str,
default=geektime_cfg,
help="specify alternate config file")
parser.add_argument("-a", "--account", dest="account", type=str,
help="specify the account phone number")
help="specify the account phone number")
parser.add_argument("-p", "--password", dest="password", type=str,
help="specify the account password")
help="specify the account password")
parser.add_argument("--area", dest="area", type=str, default='86',
help="specify the account country code")
parser.add_argument("-o", "--output-folder", dest="output_folder", type=str, default=cwd,
help="specify the output folder")
help="specify the account country code")
parser.add_argument("-o", "--output-folder", dest="output_folder", type=str,
default=cwd,
help="specify the output folder")
parser.add_argument("-c", "--course-id", dest="course_id", type=int,
help="specify the target course id")
parser.add_argument("--force", dest="force", action='store_true', default=False,
help="do not use the cache data")
parser.add_argument("--enable-comments", dest="enable_comments", action='store_true', default=False,
help="fetch the course comments")
parser.add_argument("--comments-count", dest="comments_count", type=int, default=10,
help="the count of comments to fetch each post")
parser.add_argument("--push", dest="push", action='store_true', default=False,
help="push to kindle")
parser.add_argument("--source-only", dest="source_only", action='store_true', default=False,
help="download source file only")
parser.add_argument("--url-only", dest="url_only", action='store_true', default=False,
help="specify the target course id")
parser.add_argument("--force", dest="force", action='store_true',
default=False,
help="do not use the cache data")
parser.add_argument("--enable-comments", dest="enable_comments",
action='store_true', default=False,
help="fetch the course comments")
parser.add_argument("--comments-count", dest="comments_count", type=int,
default=10,
help="the count of comments to fetch each post")
parser.add_argument("--push", dest="push", action='store_true',
default=False,
help="push to kindle")
parser.add_argument("--source-only", dest="source_only",
action='store_true', default=False,
help="download source file only")
parser.add_argument("--url-only", dest="url_only", action='store_true',
default=False,
help="download mp3/mp4 url only")
parser.add_argument("--hd-only", dest="hd_only", action='store_true', default=False,
parser.add_argument("--hd-only", dest="hd_only", action='store_true',
default=False,
help="download mp4 with high quality")
parser.add_argument("--all", dest="all", action='store_true', default=False,
help="fetch all courses")
parser.add_argument("--course-ids", dest="course_ids", type=str,
help="specify the target course ids")
parser.add_argument( "--smtp-host", dest="smtp_host", type=str,
parser.add_argument("--all", dest="all", action='store_true', default=False,
help="fetch all courses")
parser.add_argument("--course-ids", dest="course_ids", type=str,
help="specify the target course ids")
parser.add_argument("--smtp-host", dest="smtp_host", type=str,
help="specify the smtp host")
parser.add_argument("--smtp-port", dest="smtp_port", type=int,
help="specify the a smtp port")
Expand All @@ -147,7 +160,8 @@ def parse_config(args: list) -> dict:

keys = ['config', 'account', 'area', 'password', 'output_folder',
'force', 'enable_comments', 'comments_count', 'push',
'course_id', 'source_only', 'url_only', 'hd_only', 'all', 'course_ids',
'course_id', 'source_only', 'url_only', 'hd_only', 'all',
'course_ids',
'smtp_host', 'smtp_port', 'smtp_user', 'smtp_password',
'smtp_encryption', 'email_to', 'workers']
for name in keys:
Expand All @@ -169,7 +183,6 @@ def main():
command = args[0]
args = args[1:]


if command in commands:
o = commands[command]()
try:
Expand All @@ -178,4 +191,4 @@ def main():
sys.stderr.write("ERROR: {}\n".format(e))
logger.error('ERROR: {}'.format(traceback.format_exc()))
else:
print('Unknow command %r\n\n' % (command,))
sys.stderr.write('Unknow command %r\n\n' % (command,))
68 changes: 44 additions & 24 deletions geektime_dl/cli/ebook.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# coding=utf8
# flake8: noqa

import os
import sys
Expand All @@ -14,7 +15,8 @@
class EBook(Command):
"""将专栏文章制作成电子书
geektime ebook -c <course_id> [--output-folder=<output_folder>] [--enable-comments] [--comments-count=<comments_count>]
geektime ebook -c <course_id> [--output-folder=<output_folder>] \
[--enable-comments] [--comments-count=<comments_count>]
`[]`表示可选，`<>`表示相应变量值
Expand Down Expand Up @@ -54,27 +56,29 @@ def _render_source_files(self, course_intro: dict, course_content: list,
render = Render(_out_dir)
# introduction
if not force and os.path.isfile(os.path.join(_out_dir, '简介.html')):
sys.stdout.write( '{}简介 exists\n'.format(column_title))
sys.stdout.write('{}简介 exists\n'.format(column_title))
else:
render.render_article_html('简介', course_intro['column_intro'])
sys.stdout.write('下载{}简介 done\n'.format(column_title))
# cover
if not force and os.path.isfile(os.path.join(_out_dir, 'cover.jpg')):
sys.stdout.write( '{}封面 exists\n'.format(column_title))
sys.stdout.write('{}封面 exists\n'.format(column_title))
else:
render.generate_cover_img(course_intro['column_cover'])
sys.stdout.write('下载{}封面 done\n'.format(column_title))
# toc
ebook_name = self._title(course_intro)
render.render_toc_md(
ebook_name,
['简介'] + [render.format_file_name(t['article_title']) for t in articles]
['简介']
+ [render.format_file_name(t['article_title']) for t in articles]
)
sys.stdout.write('下载{}目录 done\n'.format(column_title))
# articles
for article in articles:
title = render.format_file_name(article['article_title'])
if not force and os.path.isfile(os.path.join(_out_dir, '{}.html'.format(title))):
fn = os.path.join(_out_dir, '{}.html'.format(title))
if not force and os.path.isfile(fn):
sys.stdout.write(title + ' exists\n')
continue
render.render_article_html(title, article['article_content'])
Expand All @@ -92,38 +96,50 @@ def run(self, cfg: dict) -> None:
try:
os.makedirs(out_dir)
except OSError:
sys.stderr.write("ERROR: couldn't create the output folder {}\n".format(out_dir))
sys.stderr.write(
"ERROR: couldn't create the output folder {}\n".format(
out_dir)
)
return
try:
dc = get_data_client(cfg)
except:
sys.stderr.write("ERROR: invalid geektime account or password\n"
"Use '%s login --help' for help.\n" % sys.argv[0].split(os.path.sep)[-1])
except Exception:
sys.stderr.write(
"ERROR: invalid geektime account or password\n"
"Use '{} login --help' for help.\n".format(
sys.argv[0].split(os.path.sep)[-1])
)
return

course_data = dc.get_course_intro(course_id, force=True)
if int(course_data['column_type']) not in (1, 2):
sys.stderr.write("ERROR: 该课程不提供文本:%s" % course_data['column_title'])
sys.stderr.write("ERROR: 该课程不提供文本:{}".format(
course_data['column_title']))
return

# data
sys.stdout.write('doing ......\n')
data = dc.get_course_content(course_id, force=cfg['force'])
if cfg['enable_comments']:
for post in data:
post['article_content'] += self._render_comment_html(post['comments'], cfg['comments_count'])
post['article_content'] += self._render_comment_html(
post['comments'], cfg['comments_count'])

# source file
course_data['column_title'] = Render.format_file_name(course_data['column_title'])
self._render_source_files(course_data, data, out_dir, force=cfg['force'])
course_data['column_title'] = Render.format_file_name(
course_data['column_title'])
self._render_source_files(
course_data, data, out_dir, force=cfg['force'])

# ebook
ebook_name = self._title(course_data)
if not cfg['source_only']:
if course_data['is_finish'] and os.path.isfile(os.path.join(out_dir, ebook_name) + '.mobi'):
fn = os.path.join(out_dir, ebook_name) + '.mobi'
if course_data['is_finish'] and os.path.isfile(fn):
sys.stdout.write("{} exists\n".format(ebook_name))
else:
make_mobi(source_dir=os.path.join(out_dir, course_data['column_title']), output_dir=out_dir)
src_dir = os.path.join(out_dir, course_data['column_title'])
make_mobi(source_dir=src_dir, output_dir=out_dir)

# push to kindle
if cfg['push'] and not cfg['source_only']:
Expand All @@ -132,13 +148,15 @@ def run(self, cfg: dict) -> None:
send_to_kindle(fn, cfg)
sys.stdout.write("push to kindle done\n")
except Exception as e:
sys.stderr.write("ERROR: push to kindle failed, e={}\n".format(e))
sys.stderr.write(
"ERROR: push to kindle failed, e={}\n".format(e))

@staticmethod
def _timestamp2str(timestamp: int) -> str:
if not timestamp:
return ''
return datetime.datetime.fromtimestamp(int(timestamp)).strftime("%Y-%m-%d %H:%M:%S")
return datetime.datetime.fromtimestamp(
int(timestamp)).strftime("%Y-%m-%d %H:%M:%S")

def _render(self, c):
replies = json.loads(c.get('replies'))
Expand All @@ -155,6 +173,7 @@ def _render(self, c):
reply.get('content')
) if reply else ''

likes = "[{}赞]".format(c['like_count']) if c['like_count'] else ''
c_html = """
<li>
<div>
Expand All @@ -169,7 +188,7 @@ def _render(self, c):
</li>
""".format(
user_name=c['user_name'],
like_count="[{}赞]".format(c['like_count']) if c['like_count'] else '',
like_count=likes,
comment_content=c['comment_content'],
comment_time=self._timestamp2str(c['comment_ctime']),
replies=replies_html
Expand Down Expand Up @@ -206,18 +225,19 @@ def run(self, cfg: dict):
if cfg['all']:
try:
dc = get_data_client(cfg)
except:
sys.stderr.write("ERROR: invalid geektime account or password\n"
"Use '{} login --help' for help.\n".format(
sys.argv[0].split(os.path.sep)[-1]))
except Exception:
sys.stderr.write(
"ERROR: invalid geektime account or password\n"
"Use '{} login --help' for help.\n".format(
sys.argv[0].split(os.path.sep)[-1]))
return

data = dc.get_course_list()
for c in data['1']['list'] + data['2']['list']:
if c['had_sub'] and (c['update_frequency'] == '全集' or c['is_finish']):
if (c['had_sub']
and (c['update_frequency'] == '全集' or c['is_finish'])):
cid_list.append(c['id'])


else:
course_ids = cfg['course_ids']
cid_list.extend(course_ids.split(','))
Expand Down
7 changes: 5 additions & 2 deletions geektime_dl/cli/login.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

class Login(Command):
"""登录极客时间，保存账号密码至配置文件
geektime login [--account=<account>] [--password=<password>] [--area=<area>]
geektime login [--account=<account>] \
[--password=<password>] [--area=<area>]
`[]`表示可选，`<>`表示相应变量值
Expand All @@ -34,7 +35,9 @@ def run(self, args: dict):
"enter password: ".format(area, account))

if need_save:
args.update({'account': account, 'password': password, 'area': area})
args.update({
'account': account, 'password': password, 'area': area
})
save_cfg(args)

GkApiClient(account=account, password=password, area=area)
Expand Down
Loading

0 comments on commit d697495

Please sign in to comment.