Add linter: Make sure we explicitly open all text files using UTF-8

Summary:
...encoding in Python.

Backport of core PR13440 and PR13448
https://github.com/bitcoin/bitcoin/pull/13440/files
https://github.com/bitcoin/bitcoin/pull/13448/files

Depends on D3756 and D3759.

The linter has been converted to PHP and integrated into arcanist within
the same diff, as it is simple enough. It also gets improved to behave
correcty when the arguments are spead over multiple lines. It also graps
the `open()` call nested in another function call.

Test Plan:
  ./test/functional/test_runner.py
  arc lint --everything

Find an ` open()` occurrence in any python file and change the encoding
to `encoding='foo'`.
  arc lint -- <the edited file>
Check the linter returns an error where the encoding has been changed.

Reviewers: #bitcoin_abc, deadalnix

Reviewed By: #bitcoin_abc, deadalnix

Differential Revision: https://reviews.bitcoinabc.org/D3772

.arclint

@@ -131,6 +131,10 @@
         "check-rpc-mappings": {
             "type": "check-rpc-mappings",
             "include": "(^src/(rpc/|wallet/rpc).*\\.cpp$)"
+        },
+        "lint-python-encoding": {
+            "type": "lint-python-encoding",
+            "include": "(\\.py$)"
         }
     }
 }

arcanist/.phutil_module_cache

@@ -1 +1 @@
-{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}}}
+{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}}}

arcanist/__phutil_library_map__.php

@@ -26,6 +26,7 @@
     'IncludeSourceLinter' => 'linter/IncludeSourceLinter.php',
     'LocaleDependenceLinter' => 'linter/LocaleDependenceLinter.php',
     'PythonFormatLinter' => 'linter/PythonFormatLinter.php',
+    'PythonOpenFileEncodingLinter' => 'linter/PythonOpenFileEncodingLinter.php',
     'StdintLinter' => 'linter/StdintLinter.php',
     'TestsLinter' => 'linter/TestsLinter.php',
   ),
@@ -50,6 +51,7 @@
     'IncludeSourceLinter' => 'ArcanistLinter',
     'LocaleDependenceLinter' => 'ArcanistLinter',
     'PythonFormatLinter' => 'ArcanistExternalLinter',
+    'PythonOpenFileEncodingLinter' => 'ArcanistLinter',
     'StdintLinter' => 'ArcanistLinter',
     'TestsLinter' => 'ArcanistExternalLinter',
   ),

arcanist/linter/PythonOpenFileEncodingLinter.php

@@ -0,0 +1,70 @@
+<?php
+
+/**
+ * Make sure we explicitly set the encoding when opening text files with
+ * Python.
+ */
+final class PythonOpenFileEncodingLinter extends ArcanistLinter {
+
+  const ENCODING_NOT_FOUND = 1;
+
+  public function getInfoName() {
+    return 'lint-python-encoding';
+  }
+
+  public function getInfoDescription() {
+    return pht('Make sure we explicitly set the encoding when opening text '.
+               'files with Python');
+  }
+
+  public function getLinterName() {
+    return 'PYTHON_ENCODING';
+  }
+
+  public function getLinterConfigurationName() {
+    return 'lint-python-encoding';
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::ENCODING_NOT_FOUND => ArcanistLintSeverity::SEVERITY_ERROR,
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::ENCODING_NOT_FOUND => pht('Encoding should be specified when '.
+                                      'opening a text file.'),
+    );
+  }
+
+  public function lintPath($path) {
+    $path = Filesystem::resolvePath($path, $this->getProjectRoot());
+    $fileContent = Filesystem::readFile($path);
+
+    $pattern = "/[(\s]open(\(((?>[^()]+)|(?1))*\))/";
+    if (!preg_match_all($pattern, $fileContent, $matches,
+      PREG_OFFSET_CAPTURE)) {
+      return;
+    }
+
+    foreach ($matches[0] as $match) {
+      list($open, $offset) = $match;
+
+      $isBin = preg_match("/open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]/", $open);
+      $hasEncoding = preg_match("/encoding=.(ascii|utf8|utf-8)./", $open);
+
+      if (!$isBin && !$hasEncoding) {
+        $this->raiseLintAtOffset(
+          $offset + 1,
+          self::ENCODING_NOT_FOUND,
+          pht("Python's open(...) seems to be used to open text files without ".
+              "explicitly specifying encoding, or with an invalid encoding. ".
+              "Encoding should be 'ascii', 'utf-8' or 'utf8' (e.g.: ".
+              "`open(f, 'r', encoding='utf-8')`)."),
+          substr($open, 1),
+          null);
+      }
+    }
+  }
+}

contrib/devtools/circular-dependencies.py

@@ -39,7 +39,7 @@ def module_name(path):
 # TODO: implement support for multiple include directories
 for arg in sorted(files.keys()):
     module = files[arg]
-    with open(arg, 'r') as f:
+    with open(arg, 'r', encoding="utf8") as f:
         for line in f:
             match = RE.match(line)
             if match:

contrib/devtools/copyright_header.py

@@ -160,7 +160,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name):
 
 
 def read_file(filename):
-    return open(os.path.abspath(filename), 'r').read()
+    return open(os.path.abspath(filename), 'r', encoding="utf8").read()
 
 
 def gather_file_info(filename):
@@ -352,14 +352,14 @@ def get_most_recent_git_change_year(filename):
 
 
 def read_file_lines(filename):
-    f = open(os.path.abspath(filename), 'r')
+    f = open(os.path.abspath(filename), 'r', encoding="utf8")
     file_lines = f.readlines()
     f.close()
     return file_lines
 
 
 def write_file_lines(filename, file_lines):
-    f = open(os.path.abspath(filename), 'w')
+    f = open(os.path.abspath(filename), 'w', encoding="utf8")
     f.write(''.join(file_lines))
     f.close()
 

contrib/devtools/test-security-check.py

@@ -11,7 +11,7 @@
 
 
 def write_testcode(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding="utf8") as f:
         f.write('''
     #include <stdio.h>
     int main()

contrib/filter-lcov.py

@@ -16,8 +16,8 @@
 outfile = args.outfile
 
 in_remove = False
-with open(tracefile, 'r') as f:
-    with open(outfile, 'w') as wf:
+with open(tracefile, 'r', encoding="utf8") as f:
+    with open(outfile, 'w', encoding="utf8") as wf:
         for line in f:
             for p in pattern:
                 if line.startswith("SF:") and p in line:

contrib/linearize/linearize-data.py

@@ -85,7 +85,7 @@ def get_blk_dt(blk_hdr):
 
 def get_block_hashes(settings):
     blkindex = []
-    f = open(settings['hashlist'], "r")
+    f = open(settings['hashlist'], "r", encoding="utf8")
     for line in f:
         line = line.rstrip()
         if settings['rev_hash_bytes'] == 'true':
@@ -279,7 +279,7 @@ def run(self):
         print("Usage: linearize-data.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1])
+    f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
         m = re.search(r'^\s*#', line)

contrib/linearize/linearize-hashes.py

@@ -102,7 +102,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
 
 def get_rpc_cookie():
     # Open the cookie file
-    with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f:
+    with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f:
         combined = f.readline()
         combined_split = combined.split(":")
         settings['rpcuser'] = combined_split[0]
@@ -114,7 +114,7 @@ def get_rpc_cookie():
         print("Usage: linearize-hashes.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1])
+    f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
         m = re.search(r'^\s*#', line)

contrib/seeds/generate-seeds.py

@@ -133,10 +133,10 @@ def main():
     g.write(
         ' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n')
     g.write(' */\n')
-    with open(os.path.join(indir, 'nodes_main.txt'), 'r') as f:
+    with open(os.path.join(indir, 'nodes_main.txt'), 'r', encoding="utf8") as f:
         process_nodes(g, f, 'pnSeed6_main', 8333)
     g.write('\n')
-    with open(os.path.join(indir, 'nodes_test.txt'), 'r') as f:
+    with open(os.path.join(indir, 'nodes_test.txt'), 'r', encoding="utf8") as f:
         process_nodes(g, f, 'pnSeed6_test', 18333)
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
 

share/qt/extract_strings_qt.py

@@ -67,7 +67,7 @@ def parse_po(text):
 
 messages = parse_po(out.decode('utf-8'))
 
-f = open(OUT_CPP, 'w')
+f = open(OUT_CPP, 'w', encoding="utf8")
 f.write("""
 
 #include <QtGlobal>

test/functional/combine_logs.py

@@ -69,7 +69,7 @@ def get_log_events(source, logfile):
     Log events may be split over multiple lines. We use the timestamp
     regex match as the marker for a new log event."""
     try:
-        with open(logfile, 'r') as infile:
+        with open(logfile, 'r', encoding='utf-8') as infile:
             event = ''
             timestamp = ''
             for line in infile:

test/functional/feature_notifications.py

@@ -43,7 +43,7 @@ def run_test(self):
             self.block_filename).st_size >= (block_count * 65), timeout=10)
 
         # file content should equal the generated blocks hashes
-        with open(self.block_filename, 'r') as f:
+        with open(self.block_filename, 'r', encoding="utf-8") as f:
             assert_equal(sorted(blocks), sorted(f.read().splitlines()))
 
         self.log.info("test -walletnotify")
@@ -54,7 +54,7 @@ def run_test(self):
         # file content should equal the generated transaction hashes
         txids_rpc = list(
             map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count)))
-        with open(self.tx_filename, 'r') as f:
+        with open(self.tx_filename, 'r', encoding="ascii") as f:
             assert_equal(sorted(txids_rpc), sorted(f.read().splitlines()))
         os.remove(self.tx_filename)
 
@@ -69,7 +69,7 @@ def run_test(self):
         # file content should equal the generated transaction hashes
         txids_rpc = list(
             map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count)))
-        with open(self.tx_filename, 'r') as f:
+        with open(self.tx_filename, 'r', encoding="ascii") as f:
             assert_equal(sorted(txids_rpc), sorted(f.read().splitlines()))
 
         # Create an invalid chain and ensure the node warns.

test/functional/interface_zmq.py

@@ -48,7 +48,7 @@ def setup_nodes(self):
 
         # Check that bitcoin has been built with ZMQ enabled.
         config = configparser.ConfigParser()
-        config.read_file(open(self.options.configfile))
+        config.read_file(open(self.options.configfile, encoding='utf-8'))
 
         if not config["components"].getboolean("ENABLE_ZMQ"):
             raise SkipTest("bitcoind has not been built with zmq enabled.")

test/functional/rpc_getblockstats.py

@@ -96,12 +96,12 @@ def generate_test_data(self, filename):
             'mocktime': int(mocktime),
             'stats': self.expected_stats,
         }
-        with open(filename, 'w') as f:
+        with open(filename, 'w', encoding="utf8") as f:
             json.dump(to_dump, f, sort_keys=True,
                       indent=2, default=EncodeDecimal)
 
     def load_test_data(self, filename):
-        with open(filename, 'r') as f:
+        with open(filename, 'r', encoding="utf8") as f:
             d = json.load(f, parse_float=decimal.Decimal)
             blocks = d['blocks']
             mocktime = d['mocktime']