task: update linters and apply fixes (fstring usage, encoding in opens)

.travis.yml

@@ -6,12 +6,11 @@ python:
 cache: pip
 
 install:
+  - pip install -r lint_requirements.txt
   - pip install -r requirements.txt
 
 jobs:
   include:
     - stage: lint
       script:
-        - isort --check-only --line-length 140 .
-        - pycodestyle --config=.pycodestyle -- .
-        - pylint --load-plugins pylint_quotes -- *.py
+        - sh ./lint.sh

NodeConfigurator.py

@@ -19,14 +19,14 @@ def to_short_name(filename):
 
 def select_random_peers(source, destination, count):
     all_nodes = None
-    with open(source, 'r') as input_file:
+    with open(source, 'rt', encoding='utf8') as input_file:
         all_nodes = json.load(input_file)
 
     random.shuffle(all_nodes['knownPeers'])
     num_nodes = min(len(all_nodes['knownPeers']), count)
     all_nodes['knownPeers'] = all_nodes['knownPeers'][:num_nodes]
 
-    with open(destination, 'w') as output_file:
+    with open(destination, 'wt', encoding='utf8') as output_file:
         json.dump(all_nodes, output_file, indent=4)
 
 
@@ -75,12 +75,12 @@ def check_requirements(self):
         expected_names = set(['ca.pubkey.pem', 'node.full.crt.pem', 'node.key.pem', 'node.crt.pem', 'ca.crt.pem'])
 
         if names != expected_names:
-            raise RuntimeError('expecting following files in certificates directory: {}'.format(expected_names))
+            raise RuntimeError(f'expecting following files in certificates directory: {expected_names}')
 
     def unzip_nemesis(self):
         log.info('extracting nemesis seed')
-        nemesis_package = ZipFile(self.dir / 'nemesis-seed.zip')
-        nemesis_package.extractall(self.dir / 'nemesis')
+        with ZipFile(self.dir / 'nemesis-seed.zip') as nemesis_package:
+            nemesis_package.extractall(self.dir / 'nemesis')
 
     def prepare_resources(self):
         log.info('preparing base settings')
@@ -128,8 +128,8 @@ def prepare_peers(self):
         num_peers = 20
         destination = self.dir / 'resources'
         for role in ['api', 'p2p']:
-            filename = 'peers-{}.json'.format(role)
-            select_random_peers(self.templates / 'all-{}'.format(filename), destination / filename, num_peers)
+            filename = f'peers-{role}.json'
+            select_random_peers(self.templates / f'all-{filename}', destination / filename, num_peers)
 
     def create_subdir(self, dir_name):
         dir_path = self.dir / dir_name
@@ -140,8 +140,8 @@ def create_subdir(self, dir_name):
 
     def unzip_mongo_scripts(self):
         log.info('extracting mongo scripts')
-        nemesis_package = ZipFile(self.dir / 'mongo-scripts.zip')
-        nemesis_package.extractall(self.dir / 'mongo')
+        with ZipFile(self.dir / 'mongo-scripts.zip') as nemesis_package:
+            nemesis_package.extractall(self.dir / 'mongo')
 
     def prepare_startup_files(self):
         docker_compose_path = self.dir / 'docker-compose.yml'
@@ -189,10 +189,10 @@ def move_voting_key_file(self):
 
         for _, filepath in sorted(matching_names.items()):
             free_id = self.find_next_free_id()
-            destination_filename = 'private_key_tree{}.dat'.format(free_id)
+            destination_filename = f'private_key_tree{free_id}.dat'
             destination_filepath = destination_directory / destination_filename
             shutil.move(filepath, destination_filepath)
-            log.info('moving {} -> {}'.format(filepath, destination_filepath))
+            log.info(f'moving {filepath} -> {destination_filepath}')
 
     def run(self):
         self.check_requirements()

certtool.py

@@ -16,18 +16,17 @@
 
 def run_openssl(args, show_output=True):
     command_line = ['openssl'] + args
-    process = Popen(command_line, stdout=PIPE, stderr=STDOUT)
-
     all_lines = []
-    for line_bin in iter(process.stdout.readline, b''):
-        line = line_bin.decode('ascii')
-        all_lines.append(line)
+    with Popen(command_line, stdout=PIPE, stderr=STDOUT) as process:
+        for line_bin in iter(process.stdout.readline, b''):
+            line = line_bin.decode('ascii')
+            all_lines.append(line)
 
-        if show_output:
-            sys.stdout.write(line)
-            sys.stdout.flush()
+            if show_output:
+                sys.stdout.write(line)
+                sys.stdout.flush()
 
-    process.wait()
+        process.wait()
 
     return all_lines
 
@@ -36,27 +35,27 @@ def check_openssl_version():
     version_output = ''.join(run_openssl(['version', '-v'], False))
     match = re.match(r'^OpenSSL +([^ ]*) ', version_output)
     if not match or not match.group(1).startswith('1.1.1'):
-        raise RuntimeError('{} requires openssl version >=1.1.1'.format(__file__))
+        raise RuntimeError(f'{__file__} requires openssl version >=1.1.1')
 
 
 def get_common_name(default_value, prompt):
     if default_value:
         return default_value
 
-    return input('Enter {}: '.format(prompt)).strip()
+    return input(f'Enter {prompt}: ').strip()
 
 
 def prepare_ca_config(ca_pem_path, ca_cn):
-    with open('ca.cnf', 'wt') as output_file:
-        output_file.write('''[ca]
+    with open('ca.cnf', 'wt', encoding='utf8') as output_file:
+        output_file.write(f'''[ca]
 default_ca = CA_default
 
 [CA_default]
 new_certs_dir = ./new_certs
 
 database = index.txt
 serial   = serial.dat
-private_key = {private_key_path}
+private_key = {ca_pem_path}
 certificate = ca.crt.pem
 policy = policy_catapult
 
@@ -68,24 +67,24 @@ def prepare_ca_config(ca_pem_path, ca_cn):
 distinguished_name = dn
 
 [dn]
-CN = {cn}
-'''.format(private_key_path=ca_pem_path, cn=ca_cn))
+CN = {ca_cn}
+''')
 
     os.makedirs('new_certs')
     os.chmod('new_certs', 0o700)
 
-    with open('index.txt', 'wt') as output_file:
+    with open('index.txt', 'wt', encoding='utf8') as output_file:
         output_file.write('')
 
 
 def prepare_node_config(node_cn):
-    with open('node.cnf', 'wt') as output_file:
-        output_file.write('''[req]
+    with open('node.cnf', 'wt', encoding='utf8') as output_file:
+        output_file.write(f'''[req]
 prompt = no
 distinguished_name = dn
 [dn]
-CN = {cn}
-'''.format(cn=node_cn))
+CN = {node_cn}
+''')
 
 
 def main():
@@ -100,7 +99,7 @@ def main():
     filepath = Path(args.output)
     if filepath.exists():
         if not args.force:
-            raise FileExistsError('output directory ({}) already exists, use --force to overwrite'.format(filepath))
+            raise FileExistsError(f'output directory ({filepath}) already exists, use --force to overwrite')
         shutil.rmtree(filepath)
 
     check_openssl_version()
@@ -154,7 +153,7 @@ def main():
         '-in', 'node.csr.pem',
         '-out', 'node.crt.pem'])
 
-    log.info('certificates generated in {} directory'.format(args.output))
+    log.info(f'certificates generated in {args.output} directory')
 
 
 if __name__ == '__main__':

compose.py

@@ -8,18 +8,18 @@
 
 
 def patch_compose(filepath, patch_cb):
-    with open(filepath, 'r') as input_file:
+    with open(filepath, 'rt', encoding='utf8') as input_file:
         docker_compose = yaml.load(input_file, Loader=yaml.SafeLoader)
 
     patch_cb(docker_compose)
 
-    with open(filepath, 'w') as output_file:
+    with open(filepath, 'wt', encoding='utf8') as output_file:
         output_file.write(yaml.dump(docker_compose, default_style='\'', sort_keys=False))
 
 
 def patch_user(compose, container_names):
     if not sys.platform.startswith('win'):
-        user_entry = '{}:{}'.format(os.getuid(), os.getgid())
+        user_entry = f'{os.getuid()}:{os.getgid()}'
         for container in container_names:
             compose['services'][container]['user'] = user_entry
 

downloader.py

@@ -17,22 +17,23 @@ def calculate_file_hash(filepath):
 
 
 def download_file(output_dir, descriptor):
-    output_path = output_dir / descriptor['name']
+    descriptor_name = descriptor['name']
+    output_path = output_dir / descriptor_name
 
     if output_path.is_file():
         if descriptor['hash'] == calculate_file_hash(output_path):
-            log.info('proper file already downloaded ({})'.format(descriptor['name']))
+            log.info(f'proper file already downloaded ({descriptor_name})')
             return
 
         log.warn('file exists, but has invalid hash, re-downloading')
         os.remove(output_path)
 
     req = requests.get(descriptor['url'])
     if 200 != req.status_code:
-        raise RuntimeError('could not download file ({}), try again'.format(descriptor['name']))
+        raise RuntimeError(f'could not download file ({descriptor_name}), try again')
 
     if descriptor['hash'] != calculate_buffer_hash(req.content):
-        raise RuntimeError('downloaded file ({}) has invalid hash'.format(descriptor['name']))
+        raise RuntimeError(f'downloaded file ({descriptor_name}) has invalid hash')
 
     with open(output_path, 'wb') as output_file:
         output_file.write(req.content)

lint.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
+set -ex
+
 find . -type f -name "*.sh" -print0 | xargs -0 shellcheck
-find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 "$(which isort)" --check-only --line-length 140
-find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 "$(which pycodestyle)" --config=.pycodestyle
-find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 "$(which pylint)" --load-plugins pylint_quotes
+find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 python3 -m isort --check-only --line-length 140
+find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 python3 -m pycodestyle --config=.pycodestyle
+find . -type f -name "*.py" -print0 | PYTHONPATH=. xargs -0 python3 -m pylint --load-plugins pylint_quotes

lint_requirements.txt

@@ -0,0 +1,4 @@
+isort==5.10.1
+pycodestyle==2.8.0
+pylint==2.11.1
+pylint-quotes==0.2.3

patcher.py

@@ -6,10 +6,10 @@ def patch_config(root, name, patch_cb, **kwargs):
     config = RawConfigParser(comment_prefixes=None, empty_lines_in_values=False, allow_no_value=True)
 
     config.optionxform = lambda option: option
-    filename = '{}/resources/config-{}.properties'.format(root, name)
+    filename = f'{root}/resources/config-{name}.properties'
     config.read(filename)
 
     patch_cb(config, **kwargs)
 
-    with open(filename, 'wt') as output_file:
+    with open(filename, 'wt', encoding='utf8') as output_file:
         config.write(output_file)

pemtool.py

@@ -10,7 +10,7 @@
 
 
 def read_key(filename):
-    return open(filename).read()
+    return open(filename, 'rt', encoding='utf8').read()
 
 
 def get_private_key(filename):
@@ -32,20 +32,20 @@ def main():
 
     filepath = Path(output_name + '.pem')
     if filepath.exists() and not args.force:
-        raise FileExistsError('output file ({}) already exists, use --force to overwrite'.format(filepath))
+        raise FileExistsError(f'output file ({filepath}) already exists, use --force to overwrite')
 
     private_key = PrivateKey(unhexlify(get_private_key(args.input)))
 
     password = None
     if args.ask_pass:
-        password = getpass.getpass('Provide {} password: '.format(filepath))
-        confirmation = getpass.getpass('Confirm {} password: '.format(filepath))
+        password = getpass.getpass(f'Provide {filepath} password: ')
+        confirmation = getpass.getpass(f'Confirm {filepath} password: ')
         if confirmation != password:
             raise RuntimeError('Provided passwords do not match')
 
     storage = PrivateKeyStorage('.', password)
     storage.save(output_name, private_key)
-    print('saved {}'.format(filepath))
+    print(f'saved {filepath}')
 
 
 if __name__ == '__main__':

requirements.txt

@@ -1,10 +1,5 @@
-aiodns==2.0.0
-aiohttp==3.7.4
-isort==5.7.0
-pycodestyle==2.6.0
-pylint==2.7.2
-pylint-quotes==0.2.1
-pyyaml==5.4.1
-requests==2.25.1
-symbol-sdk-core-python==2.0.0
+aiohttp==3.8.1
+PyYAML==5.4.1
+requests==2.26.0
+symbol-sdk-core-python==2.0.1
 zenlog==1.1

settings.py

@@ -9,11 +9,11 @@
 
 
 def get_txt_key(filename):
-    return open(filename + '.txt').read().strip()
+    return open(filename + '.txt', 'rt', encoding='utf8').read().strip()
 
 
 def get_pem_key(ask_pass, filename):
-    password = getpass.getpass('Provide {}.pem password: '.format(filename)) if ask_pass else None
+    password = getpass.getpass(f'Provide {filename}.pem password: ') if ask_pass else None
     storage = PrivateKeyStorage('.', password)
     return str(storage.load(filename))
 
@@ -69,8 +69,8 @@ def check_harvesting_files():
     if Path(HARVESTING_KEY_FILENAME + '.txt').is_file() and Path(VRF_KEY_FILENAME + '.txt').is_file():
         return
 
-    raise RuntimeError('harvesting requested, but harvesting or vrf key files do not exist ({}, {}) (.pem or .txt)'.format(
-        HARVESTING_KEY_FILENAME, VRF_KEY_FILENAME))
+    raise RuntimeError(f'harvesting requested, but harvesting or vrf key files do not exist ({HARVESTING_KEY_FILENAME}, {VRF_KEY_FILENAME})'
+                       ' (.pem or .txt)')
 
 
 def patch_harvesting(config, **kwargs):

votingkey.py

@@ -20,7 +20,7 @@
 async def fetch_html(url, session, **kwargs):
     resp = await session.request(method='GET', url=url, **kwargs)
     resp.raise_for_status()
-    log.info('Got response [{}] for URL: {}'.format(resp.status, url))
+    log.info(f'Got response [{resp.status}] for URL: {url}')
     data = await resp.json()
     return data
 
@@ -30,9 +30,12 @@ async def parse(url, session, collected):
         data = await fetch_html(url, session)
         collected[url] = data
     except (aiohttp.ClientError, aiohttp.http_exceptions.HttpProcessingError) as exc:
-        log.error('aiothtp exception for {} [{}]: {}'.format(url, getattr(exc, 'status', None), getattr(exc, 'message', None)))
+        status = getattr(exc, 'status', None)
+        message = getattr(exc, 'message', None)
+        log.error(f'aiothtp exception for {url} [{status}]: {message}')
     except Exception as exc:  # pylint: disable=broad-except
-        log.error('Non-aiohttp exception occured: {}'.format(getattr(exc, '__dict__', {})))
+        exception_attributes = getattr(exc, '__dict__', {})
+        log.error(f'Non-aiohttp exception occured: {exception_attributes}')
 
 
 async def get(urls, epoch_descriptor):
@@ -46,16 +49,16 @@ async def get(urls, epoch_descriptor):
 
 
 def toUrl(node):
-    return 'http://{}:3000/chain/info'.format(node)
+    return f'http://{node}:3000/chain/info'
 
 
 def generate_voting_key_file(filepath, start_epoch, epoch_range):
     key_pair = KeyPair(PrivateKey.random())
     voting_keys_generator = VotingKeysGenerator(key_pair)
     end_epoch = start_epoch + epoch_range
     voting_key_buffer = voting_keys_generator.generate(start_epoch, end_epoch)
-    log.info('voting key start epoch: {}, end epoch: {}'.format(start_epoch, end_epoch))
-    log.info('voting key root public key: {}'.format(key_pair.public_key))
+    log.info(f'voting key start epoch: {start_epoch}, end epoch: {end_epoch}')
+    log.info(f'voting key root public key: {key_pair.public_key}')
 
     # create the file
     with open(filepath, 'wb') as output_file:
@@ -83,7 +86,7 @@ def main():
 
     filepath = Path(os.getcwd()) / args.filename
     if filepath.exists() and not args.force:
-        raise FileExistsError('output file ({}) already exists, use --force to overwrite'.format(filepath))
+        raise FileExistsError(f'output file ({filepath}) already exists, use --force to overwrite')
 
     start_epoch = args.start_epoch
     if 0 == start_epoch: