Integrate band references into portable signature files (OSGeo#1501)

Imagery: untie signature files from groups

Thus far signature files have been living inside an imagery group. Mapping between individual raster maps of a group and signature values was implicit. This old design made impossible to safely re-use signatures of one imagery group to classify other group(s). The new approach is to have raster band references written inside a signature file to store mapping of signature values to rasters they represent. This change allows to safely use signature file for classification of other imagery groups as long as they contain rasters of the same semantic content (e.g. different image form the same satellite).

* On signature file creation write out raster band references
* On signature file reading compare group and signature file band references for a match
* Signature files now live outside of groups in a dedicated folder with subfolders for each signature file type
* All imagery modules are adjusted to use new signature handling functions

.github/workflows/build_centos.sh

@@ -21,6 +21,10 @@ set -u
 
 export INSTALL_PREFIX=$1
 
+# Old versions of GCC on CentOS default to C89 although are C11 capable
+# This causes compilation to fail on >C89 code
+export CFLAGS="-O2 -std=gnu11"
+
 ./configure \
     --prefix="$INSTALL_PREFIX/" \
     --without-freetype \

gui/wxpython/gui_core/forms.py

@@ -383,28 +383,6 @@ def run(self):
             elif name == "SubGroupSelect":
                 self.data[win.Insert] = {"group": p.get("value", "")}
 
-            elif name == "SignatureSelect":
-                if p.get("prompt", "group") == "group":
-                    group = p.get("value", "")
-                    pSubGroup = self.task.get_param(
-                        "subgroup", element="prompt", raiseError=False
-                    )
-                    if pSubGroup:
-                        subgroup = pSubGroup.get("value", "")
-                    else:
-                        subgroup = None
-                else:
-                    subgroup = p.get("value", "")
-                    pGroup = self.task.get_param(
-                        "group", element="prompt", raiseError=False
-                    )
-                    if pGroup:
-                        group = pGroup.get("value", "")
-                    else:
-                        group = None
-
-                self.data[win.Insert] = {"group": group, "subgroup": subgroup}
-
             elif name == "LocationSelect":
                 pDbase = self.task.get_param(
                     "dbase", element="element", raiseError=False
@@ -1756,8 +1734,16 @@ def __init__(self, parent, giface, task, id=wx.ID_ANY, frame=None, *args, **kwar
 
                 # sigrature file
                 elif prompt == "sigfile":
+                    if p.get("age", "") == "new":
+                        mapsets = [
+                            grass.gisenv()["MAPSET"],
+                        ]
+                    else:
+                        mapsets = None
                     selection = gselect.SignatureSelect(
-                        parent=which_panel, element=p.get("element", "sig")
+                        parent=which_panel,
+                        element=p.get("element", "sig"),
+                        mapsets=mapsets,
                     )
                     p["wxId"] = [selection.GetId()]
                     selection.Bind(wx.EVT_TEXT, self.OnSetValue)
@@ -2365,7 +2351,6 @@ def OnCheckItem(index, flag):
         pColumn = []
         pGroup = None
         pSubGroup = None
-        pSigFile = []
         pDbase = None
         pLocation = None
         pMapset = None
@@ -2412,8 +2397,6 @@ def OnCheckItem(index, flag):
                 pGroup = p
             elif prompt == "subgroup":
                 pSubGroup = p
-            elif prompt == "sigfile":
-                pSigFile.append(p)
             elif prompt == "dbase":
                 pDbase = p
             elif prompt == "location":
@@ -2430,9 +2413,6 @@ def OnCheckItem(index, flag):
         pLayerIds = []
         for p in pLayer:
             pLayerIds += p["wxId"]
-        pSigFileIds = []
-        for p in pSigFile:
-            pSigFileIds += p["wxId"]
         pSqlWhereIds = []
         for p in pSqlWhere:
             pSqlWhereIds += p["wxId"]
@@ -2459,11 +2439,7 @@ def OnCheckItem(index, flag):
             pTable["wxId-bind"] = pColumnIds
 
         if pGroup and pSubGroup:
-            if pSigFile:
-                pGroup["wxId-bind"] = pSigFileIds + pSubGroup["wxId"]
-                pSubGroup["wxId-bind"] = pSigFileIds
-            else:
-                pGroup["wxId-bind"] = pSubGroup["wxId"]
+            pGroup["wxId-bind"] = pSubGroup["wxId"]
 
         if pDbase and pLocation:
             pDbase["wxId-bind"] = pLocation["wxId"]

gui/wxpython/gui_core/gselect.py

@@ -47,6 +47,7 @@
 import sys
 import glob
 import six
+import ctypes
 
 import wx
 
@@ -58,6 +59,13 @@
 import grass.script as grass
 from grass.script import task as gtask
 from grass.exceptions import CalledModuleError
+from grass.lib.imagery import (
+    I_SIGFILE_TYPE_SIG,
+    I_SIGFILE_TYPE_SIGSET,
+    I_signatures_list_by_type,
+    I_free_signatures_list,
+)
+from grass.pygrass.utils import decode
 
 from gui_core.widgets import ManageSettingsWidget, CoordinatesValidator
 
@@ -2798,44 +2806,37 @@ def __init__(
         self,
         parent,
         element,
+        mapsets,
         id=wx.ID_ANY,
         size=globalvar.DIALOG_GSELECT_SIZE,
         **kwargs,
     ):
         super(SignatureSelect, self).__init__(parent, id, size=size, **kwargs)
-        self.element = element
-        self.SetName("SignatureSelect")
-
-    def Insert(self, group, subgroup=None):
-        """Insert signatures for defined group/subgroup
 
-        :param group: group name (can be fully-qualified)
-        :param subgroup: non fully-qualified name of subgroup
-        """
-        if not group:
-            return
-        gisenv = grass.gisenv()
-        try:
-            name, mapset = group.split("@", 1)
-        except ValueError:
-            name = group
-            mapset = gisenv["MAPSET"]
-
-        path = os.path.join(
-            gisenv["GISDBASE"], gisenv["LOCATION_NAME"], mapset, "group", name
-        )
-
-        if subgroup:
-            path = os.path.join(path, "subgroup", subgroup)
-        try:
-            items = list()
-            for element in os.listdir(os.path.join(path, self.element)):
-                items.append(element)
-            self.SetItems(items)
-        except OSError:
-            self.SetItems([])
+        sig_type = None
+        # Extend here if a new signature type is introduced
+        if element == "signatures/sig":
+            sig_type = I_SIGFILE_TYPE_SIG
+        elif element == "signatures/sigset":
+            sig_type = I_SIGFILE_TYPE_SIGSET
+        items = []
+        if sig_type is not None:
+            if mapsets:
+                for mapset in mapsets:
+                    self._append_mapset_signatures(mapset, sig_type, items)
+            else:
+                self._append_mapset_signatures(None, sig_type, items)
+        self.SetItems(items)
         self.SetValue("")
 
+    def _append_mapset_signatures(self, mapset, sig_type, items):
+        list_ptr = ctypes.POINTER(ctypes.c_char_p)
+        sig_list = list_ptr()
+        count = I_signatures_list_by_type(sig_type, mapset, ctypes.byref(sig_list))
+        for n in range(count):
+            items.append(decode(sig_list[n]))
+        I_free_signatures_list(count, sig_list)
+
 
 class SeparatorSelect(wx.ComboBox):
     """Widget for selecting seperator"""

gui/wxpython/iclass/dialogs.py

@@ -599,8 +599,6 @@ class IClassSignatureFileDialog(wx.Dialog):
     def __init__(
         self,
         parent,
-        group,
-        subgroup,
         file=None,
         title=_("Save signature file"),
         id=wx.ID_ANY,
@@ -610,7 +608,6 @@ def __init__(
         """Dialog for saving signature file
 
         :param parent: window
-        :param group: group name
         :param file: signature file name
         :param title: window title
         """
@@ -622,16 +619,9 @@ def __init__(
 
         # inconsistent group and subgroup name
         # path:
-        # grassdata/nc_spm_08/landsat/group/test_group/subgroup/test_group/sig/sigFile
+        # grassdata/nc_spm_08/landsat/signatures/sig/sigFile
         self.baseFilePath = os.path.join(
-            env["GISDBASE"],
-            env["LOCATION_NAME"],
-            env["MAPSET"],
-            "group",
-            group,
-            "subgroup",
-            subgroup,
-            "sig",
+            env["GISDBASE"], env["LOCATION_NAME"], env["MAPSET"], "signatures", "sig"
         )
         self.panel = wx.Panel(parent=self, id=wx.ID_ANY)
 

gui/wxpython/iclass/frame.py

@@ -1117,7 +1117,17 @@ def RunAnalysis(self):
             return False
 
         I_free_signatures(self.signatures)
-        I_iclass_init_signatures(self.signatures, self.refer)
+        ret = I_iclass_init_signatures(self.signatures, self.refer)
+        if not ret:
+            GMessage(
+                parent=self,
+                message=_(
+                    "There was an error initializing signatures. "
+                    "Check GUI console for any error messages."
+                ),
+            )
+            I_free_signatures(self.signatures)
+            return False
 
         # why create copy
         # cats = self.statisticsList[:]
@@ -1201,9 +1211,16 @@ def OnSaveSigFile(self, event):
                 qdlg.Destroy()
                 return
 
-        dlg = IClassSignatureFileDialog(
-            self, group=self.g["group"], subgroup=self.g["subgroup"], file=self.sigFile
-        )
+        if not self.signatures.contents.nsigs:
+            GMessage(
+                parent=self,
+                message=_(
+                    "Signatures are not valid. Recalculate them and then try again."
+                ),
+            )
+            return
+
+        dlg = IClassSignatureFileDialog(self, file=self.sigFile)
 
         if dlg.ShowModal() == wx.ID_OK:
             if os.path.exists(dlg.GetFileName(fullPath=True)):
@@ -1223,9 +1240,7 @@ def OnSaveSigFile(self, event):
                     qdlg.Destroy()
                     return
             self.sigFile = dlg.GetFileName()
-            self.WriteSignatures(
-                self.signatures, self.g["group"], self.g["subgroup"], self.sigFile
-            )
+            self.WriteSignatures(self.signatures, self.sigFile)
 
         dlg.Destroy()
 
@@ -1248,14 +1263,13 @@ def InitStatistics(self):
         self.refer = pointer(refer_obj)
         I_init_group_ref(self.refer)  # must be freed on exit
 
-    def WriteSignatures(self, signatures, group, subgroup, filename):
+    def WriteSignatures(self, signatures, filename):
         """Writes current signatures to signature file
 
         :param signatures: signature (c structure)
-        :param group: imagery group
         :param filename: signature file name
         """
-        I_iclass_write_signatures(signatures, group, subgroup, filename)
+        I_iclass_write_signatures(signatures, filename)
 
     def CheckInput(self, group, vector):
         """Check if input is valid"""

imagery/i.cca/i.cca.html

@@ -60,6 +60,8 @@ <h2>SEE ALSO</h2>
 
 <em>
 <a href="g.gui.iclass.html">g.gui.iclass</a>,
+<a href="i.gensig.html">i.gensig</a>,
+<a href="i.cluster.html">i.cluster</a>,
 <a href="i.pca.html">i.pca</a>,
 <a href="r.covar.html">r.covar</a>,
 <a href="r.mapcalc.html">r.mapcalc</a>
@@ -74,11 +76,13 @@ <h2>AUTHORS</h2>
 
 David Satnik, GIS Laboratory, 
 Central Washington University
-
 <br>
-
 Ali R. Vali, 
 University of Texas
+<br>
+Band reference support: Maris Nartiss,
+University of Latvia
+
 <!--
 <p>
 <i>Last changed: $Date$</i>

imagery/i.cca/main.c

@@ -71,6 +71,7 @@ int main(int argc, char *argv[])
     struct Signature sigs;
     FILE *sigfp;
     struct Ref refs;
+    char **err;
     int *datafds;
     int *outfds;
 
@@ -91,13 +92,14 @@ int main(int argc, char *argv[])
 
     grp_opt = G_define_standard_option(G_OPT_I_GROUP);
 
-    subgrp_opt = G_define_standard_option(G_OPT_I_GROUP);
+    subgrp_opt = G_define_standard_option(G_OPT_I_SUBGROUP);
     subgrp_opt->key = "subgroup";
     subgrp_opt->description = _("Name of input imagery subgroup");
 
     sig_opt = G_define_option();
     sig_opt->key = "signature";
     sig_opt->type = TYPE_STRING;
+    sig_opt->gisprompt = "old,signatures/sig,sigfile";
     sig_opt->required = YES;
     sig_opt->key_desc = "name";
     sig_opt->description = _("File containing spectral signatures");
@@ -113,16 +115,18 @@ int main(int argc, char *argv[])
     if (I_find_group(grp_opt->answer) <= 0)
 	G_fatal_error(_("Unknown imagery group."));
 
+    if (!I_find_subgroup(grp_opt->answer, subgrp_opt->answer))
+	G_fatal_error(_("Subgroup <%s> in group <%s> not found"),
+		      subgrp_opt->answer, grp_opt->answer);
+
     if (I_get_subgroup_ref(grp_opt->answer, subgrp_opt->answer, &refs) <= 0)
 	G_fatal_error(_("Unable to find subgroup reference information."));
 
     /* open and input the signatures file */
     if ((sigfp =
-	 I_fopen_signature_file_old(grp_opt->answer, subgrp_opt->answer,
-				    sig_opt->answer)) == NULL)
+	 I_fopen_signature_file_old(sig_opt->answer)) == NULL)
 	G_fatal_error(_("Unable to open the signature file"));
 
-    I_init_signatures(&sigs, refs.nfiles);
     if (I_read_signatures(sigfp, &sigs) < 0)
 	G_fatal_error(_("Error while reading the signatures file."));
 
@@ -131,6 +135,15 @@ int main(int argc, char *argv[])
     if (nclass < 2)
 	G_fatal_error(_("Need at least two signatures in signature file."));
 
+    err = I_sort_signatures_by_bandref(&sigs, &refs);
+    if (err)
+        G_fatal_error(_("Signature – group member band reference mismatch.\n"
+            "Extra signatures for bands: %s\n"
+            "Imagery group bands without signatures: %s"),
+            err[0] ? err[0] : _("none"),
+            err[1] ? err[1] : _("none")
+        );
+
     /* check the number of input bands */
     bands = refs.nfiles;
 

imagery/i.cluster/global.h

@@ -26,7 +26,8 @@ extern int mcs;
 extern char *group;
 extern char *subgroup;
 extern struct Ref ref;
-extern char *outsigfile;
+extern char **bandrefs;
+extern char outsigfile[GNAME_MAX + GMAPSET_MAX];
 extern char *insigfile;
 extern char *reportfile;
 extern DCELL **cell;