REL: FIDO v1.6.0

- bumped version to 1.6.0;
- updated RELEASENOTES.md;
- added catch and output for exception when mapping to a non-existing ID;
- fixed bug when updating signatures to a specific version; and
- signature builder outputs to single line when do now.

RELEASENOTES.md

@@ -9,27 +9,10 @@ Copyright 2010 The Open Preservation Foundation
 Fido is made available under the Apache License, Version 2.0; see the file
 LICENSE.txt for details.
 
-Fido 1.6.0rc5
+Fido 1.6.0
 -------------
 
-2022-08-03
-
-- added update signature parameter to control signature download verison:
-- trapped regex creation exception so that sig file creation is not derailed;
-- PRONOM/DROID signature file now downloaded from URL rather than via SOAP service;
-- moved sleep between SOAP downloads so that it's only applied between actual downloads, not when processing cached results;
-- code style warnings:
-  - some minor refactoring for complex methods;
-  - factoring out string constants;
-  - renamed some variables and methods;
-  - removed some commented code;
-  - tidied exit conditions; and
-  - removed some unreachable code.
-
-Fido 1.6.0rc4
--------------
-
-2022-06-22
+2022-12-15
 
 New command line options for updating signatures
 
@@ -42,6 +25,17 @@ New command line options for updating signatures
 - Closed issue [#179][], [#198][]: Crash on XLS format by updating olefile version to 0.46 [[#195][]].
 - Closed issue [#179][]: Crash on XLS format by updating olefile version to 0.46 [[#195][]].
 - Closed issue [#192][]: Fixed signature file defaults [[#193][]].
+- added update signature parameter to control signature download verison:
+- trapped regex creation exception so that sig file creation is not derailed;
+- PRONOM/DROID signature file now downloaded from URL rather than via SOAP service;
+- moved sleep between SOAP downloads so that it's only applied between actual downloads, not when processing cached results;
+- code style warnings:
+  - some minor refactoring for complex methods;
+  - factoring out string constants;
+  - renamed some variables and methods;
+  - removed some commented code;
+  - tidied exit conditions; and
+  - removed some unreachable code.
 
 [#100]: https://github.com/openpreserve/fido/issues/100
 [#150]: https://github.com/openpreserve/fido/issues/150

fido/__init__.py

@@ -14,7 +14,7 @@
 from six.moves import input as rinput
 
 
-__version__ = '1.6.0rc5'
+__version__ = '1.6.0'
 
 
 CONFIG_DIR = join(abspath(dirname(__file__)), 'conf')

fido/conf/versions.xml

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='utf-8'?>
 <versions>
-	<pronomVersion>107</pronomVersion>
-	<pronomSignature>formats-v107.xml</pronomSignature>
+	<pronomVersion>109</pronomVersion>
+	<pronomSignature>formats-v109.xml</pronomSignature>
 	<pronomContainerSignature>container-signature-20200121.xml</pronomContainerSignature>
 	<fidoExtensionSignature>format_extensions.xml</fidoExtensionSignature>
-	<updateScript>1.6.0rc5</updateScript>
+	<updateScript>1.6.0</updateScript>
 	<updateSite>https://fidosigs.openpreservation.org</updateSite>
-</versions>
+</versions>

fido/prepare.py

@@ -148,7 +148,10 @@ def load_pronom_xml(self, puid_filter=None):
                 id_map[pronom_id] = puid
             for element in formats:
                 for rel in element.findall('has_priority_over'):
-                    rel.text = id_map[rel.text]
+                    try:
+                        rel.text = id_map[rel.text]
+                    except KeyError as e:
+                        print("Error looking up priority over PRONOM ID {0} for format {1}".format(rel.text, element.find('puid').text), file=sys.stderr)
 
         self._sort_formats(formats)
         self.formats = formats

fido/update_signatures.py

@@ -147,8 +147,7 @@ def download_signatures(defaults, format_eles, resume, tmpdir):
     for format_ele in format_eles:
         download_sig(format_ele, tmpdir, resume, defaults)
         numfiles += 1
-        percent = int(float(numfiles) / one_percent)
-        print(r"{}/{} files [{}%]".format(numfiles, puid_count, percent))
+        print(r"Downloaded {}/{} files [{}%]".format(numfiles, puid_count, int(float(numfiles) / one_percent)), end="\r")
     print("100%")
 
 

fido/versions.py

@@ -128,7 +128,7 @@ def sig_file_actions(sig_act):
         _check_update_signatures(sig_vers, update_url, versions, sig_act == 'update')
     else:
         # Download a specific version of the signature file
-        _download_sig_version(sig_vers, update_url, versions)
+        _download_sig_version(sig_act, update_url, versions)
     sys.stdout.flush()
     sys.exit(0)
 
@@ -150,6 +150,7 @@ def _check_update_signatures(sig_vers, update_url, versions, is_update=False):
     sys.exit(0)
 
 def _download_sig_version(sig_act, update_url, versions):
+    sys.stdout.write('Downloading signature files for version {}\n'.format(sig_act))
     match = re.search('^v?(\d+)$', sig_act, re.IGNORECASE)
 
     if not match:
@@ -171,7 +172,7 @@ def _get_version(ver_string):
     return ver_string if not ver.startswith('v') else ver_string[1:]
 
 def _output_details(version, update_url, versions):
-    sys.stdout.write('Updating signature file.\n')
+    sys.stdout.write('Updating signature file to {}.\n'.format(version))
     _write_sigs(version, update_url, 'fido', 'formats-v{}.xml')
     _write_sigs(version, update_url, 'droid', 'DROID_SignatureFile-v{}.xml')
     _write_sigs(version, update_url, 'pronom', 'pronom-xml-v{}.zip')
@@ -193,5 +194,5 @@ def _write_sigs(latest, update_url, type, name_template):
     sig_out = str(importlib_resources.files('fido').joinpath('conf', name_template.format(latest)))
     if os.path.exists(sig_out):
         return
-    resp = requests.get(update_url + 'format/latest/{}/'.format(type))
+    resp = requests.get(update_url + 'format/{0}/{1}/'.format(latest, type))
     open(sig_out, 'wb').write(resp.content)