netcdfsgwriterutil.h

/******************************************************************************
 *
 * Project:  netCDF read/write Driver
 * Purpose:  GDAL bindings over netCDF library.
 * Author:   Winor Chen <wchen329 at wisc.edu>
 *
 ******************************************************************************
 * Copyright (c) 2019, Winor Chen <wchen329 at wisc.edu>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 ****************************************************************************/
#ifndef __NETCDFSGWRITERUTIL_H__
#define __NETCDFSGWRITERUTIL_H__
#include <cstdio>
#include <queue>
#include <typeinfo>
#include <vector>
#include "cpl_vsi.h"
#include "ogr_core.h"
#include "ogrsf_frmts.h"
#include "netcdflayersg.h"
#include "netcdfsg.h"
#include "netcdfvirtual.h"

namespace nccfdriver
{

/* OGR_SGeometry_Feature
 * Constructs over a OGRFeature
 * gives some basic information about that SGeometry Feature such as
 * Hold references... limited to scope of its references
 * - what's its geometry type
 * - how much total points it has
 * - how many parts it has
 * - a vector of counts of points for each part
 */
class SGeometry_Feature
{
    bool hasInteriorRing;
    const OGRGeometry *geometry_ref;
    geom_t type;
    size_t total_point_count;
    size_t total_part_count;
    std::vector<size_t> ppart_node_count;
    std::vector<bool> part_at_ind_interior;  // for use with Multipolygons ONLY
    mutable OGRPoint pt_buffer;

  public:
    geom_t getType()
    {
        return this->type;
    }
    size_t getTotalNodeCount()
    {
        return this->total_point_count;
    }
    size_t getTotalPartCount()
    {
        return this->total_part_count;
    }
    std::vector<size_t> &getPerPartNodeCount()
    {
        return this->ppart_node_count;
    }
    const OGRPoint &getPoint(size_t part_no, int point_index) const;
    explicit SGeometry_Feature(OGRFeature &);
    bool getHasInteriorRing()
    {
        return this->hasInteriorRing;
    }
    bool IsPartAtIndInteriorRing(size_t ind)
    {
        return this->part_at_ind_interior[ind];
    }  // ONLY used for Multipolygon
};

/* A memory buffer with a soft limit
 * Has basic capability of over quota checking, and memory counting
 */
class WBuffer
{
    unsigned long long used_mem = 0;

  public:
    /* addCount(...)
     * Takes in a size, and directly adds that size to memory count
     */
    void addCount(unsigned long long memuse);

    /* subCount(...)
     * Directly subtracts the specified size from used_mem
     */
    void subCount(unsigned long long memfree);
    unsigned long long &getUsage()
    {
        return used_mem;
    }

    void reset()
    {
        this->used_mem = 0;
    }

    WBuffer()
    {
    }
};

/* OGR_SGFS_Transaction
 * Abstract class for a committable transaction
 *
 */
class OGR_SGFS_Transaction
{
    int varId = INVALID_VAR_ID;

  public:
    /* int commit(...);
     * Arguments: int ncid, the dataset to write to
     *            int write_loc, the index in which to write to
     * Implementation: should write the transaction to netCDF file
     *
     */
    virtual void commit(netCDFVID &n, size_t write_loc) = 0;

    /* unsigned long long count(...)
     * Implementation: supposed to return an approximate count of memory usage
     * Most classes will implement with sizeof(*this), except if otherwise
     * uncounted for dynamic allocation is involved.
     */
    virtual unsigned long long count() = 0;

    /* appendToLog
     * Implementation - given a file pointer, a transaction will be written to
     * that log file in the format:
     * -
     * transactionVarId - sizeof(int) bytes
     * NC_TYPE - sizeof(int) bytes
     * (nc_char only) OP - 1 byte (0 if does not require COUNT or non-zero i.e.
     * 1 if does) (nc_char only): SIZE of data - sizeof(size_t) bytes DATA -
     * size depends on NC_TYPE
     */
    virtual void appendToLog(VSILFILE *) = 0;

    /* ~OGR_SGFS_Transaction()
     * Empty. Simply here to stop the compiler from complaining...
     */
    virtual ~OGR_SGFS_Transaction()
    {
    }

    /* OGR_SGFS_Transaction()
     * Empty. Simply here to stop one of the CI machines from complaining...
     */
    OGR_SGFS_Transaction()
    {
    }

    /* void getVarId(...);
     * Gets the var in which to commit the transaction to.
     */
    int getVarId()
    {
        return this->varId;
    }

    /* nc_type getType
     * Returns the type of transaction being saved
     */
    virtual nc_type getType() = 0;

    /* void setVarId(...);
     * Sets the var in which to commit the transaction to.
     */
    void setVarId(int vId)
    {
        this->varId = vId;
    }
};

typedef std::map<int, void *> NCWMap;
typedef std::pair<int, void *> NCWEntry;  // NC Writer Entry
typedef std::unique_ptr<OGR_SGFS_Transaction>
    MTPtr;  // a.k.a Managed Transaction Ptr

template <class T_c_type, nc_type T_nc_type>
void genericLogAppend(T_c_type r, int vId, VSILFILE *f)
{
    T_c_type rep = r;
    int varId = vId;
    int type = T_nc_type;
    VSIFWriteL(&varId, sizeof(int), 1, f);     // write varID data
    VSIFWriteL(&type, sizeof(int), 1, f);      // write NC type
    VSIFWriteL(&rep, sizeof(T_c_type), 1, f);  // write data
}

template <class T_c_type, class T_r_type>
MTPtr genericLogDataRead(int varId, VSILFILE *f)
{
    T_r_type data;
    if (!VSIFReadL(&data, sizeof(T_r_type), 1, f))
    {
        return MTPtr(nullptr);  // invalid read case
    }
    return MTPtr(new T_c_type(varId, data));
}

/* OGR_SGFS_NC_Char_Transaction
 * Writes to an NC_CHAR variable
 */
class OGR_SGFS_NC_Char_Transaction : public OGR_SGFS_Transaction
{
    std::string char_rep;

  public:
    void commit(netCDFVID &n, size_t write_loc) override
    {
        n.nc_put_vvar1_text(OGR_SGFS_Transaction::getVarId(), &write_loc,
                            char_rep.c_str());
    }
    unsigned long long count() override
    {
        return char_rep.size() + sizeof(*this);
    }  // account for actual character representation, this class
    void appendToLog(VSILFILE *f) override;
    nc_type getType() override
    {
        return NC_CHAR;
    }
    OGR_SGFS_NC_Char_Transaction(int i_varId, const char *pszVal)
        : char_rep(pszVal)
    {
        OGR_SGFS_Transaction::setVarId(i_varId);
    }
};

/* OGR_SGFS_NC_CharA_Transaction
 * Writes to an NC_CHAR variable, using vara instead of var1
 * Used to store 2D character array values, specifically
 */
class OGR_SGFS_NC_CharA_Transaction : public OGR_SGFS_Transaction
{
    std::string char_rep;
    size_t counts[2];

  public:
    void commit(netCDFVID &n, size_t write_loc) override
    {
        size_t ind[2] = {write_loc, 0};
        n.nc_put_vvara_text(OGR_SGFS_Transaction::getVarId(), ind, counts,
                            char_rep.c_str());
    }
    unsigned long long count() override
    {
        return char_rep.size() + sizeof(*this);
    }  // account for actual character representation, this class
    void appendToLog(VSILFILE *f) override;
    nc_type getType() override
    {
        return NC_CHAR;
    }
    OGR_SGFS_NC_CharA_Transaction(int i_varId, const char *pszVal)
        : char_rep(pszVal), counts{1, char_rep.length()}
    {
        OGR_SGFS_Transaction::setVarId(i_varId);
    }
};

template <class VClass, nc_type ntype>
class OGR_SGFS_NC_Transaction_Generic : public OGR_SGFS_Transaction
{
    VClass rep;

  public:
    void commit(netCDFVID &n, size_t write_loc) override
    {
        n.nc_put_vvar_generic<VClass>(OGR_SGFS_Transaction::getVarId(),
                                      &write_loc, &rep);
    }

    unsigned long long count() override
    {
        return sizeof(*this);
    }
    void appendToLog(VSILFILE *f) override
    {
        genericLogAppend<VClass, ntype>(rep, OGR_SGFS_Transaction::getVarId(),
                                        f);
    }

    OGR_SGFS_NC_Transaction_Generic(int i_varId, VClass in) : rep(in)
    {
        OGR_SGFS_Transaction::setVarId(i_varId);
    }

    VClass getData()
    {
        return rep;
    }

    nc_type getType() override
    {
        return ntype;
    }
};

typedef OGR_SGFS_NC_Transaction_Generic<signed char, NC_BYTE>
    OGR_SGFS_NC_Byte_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<short, NC_SHORT>
    OGR_SGFS_NC_Short_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<int, NC_INT>
    OGR_SGFS_NC_Int_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<float, NC_FLOAT>
    OGR_SGFS_NC_Float_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<double, NC_DOUBLE>
    OGR_SGFS_NC_Double_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<unsigned, NC_UINT>
    OGR_SGFS_NC_UInt_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<unsigned long long, NC_UINT64>
    OGR_SGFS_NC_UInt64_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<long long, NC_INT64>
    OGR_SGFS_NC_Int64_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<unsigned char, NC_UBYTE>
    OGR_SGFS_NC_UByte_Transaction;
typedef OGR_SGFS_NC_Transaction_Generic<unsigned short, NC_USHORT>
    OGR_SGFS_NC_UShort_Transaction;

/* OGR_SGFS_NC_String_Transaction
 * Writes to an NC_STRING variable, in a similar manner as NC_Char
 */
class OGR_SGFS_NC_String_Transaction : public OGR_SGFS_Transaction
{
    std::string char_rep;

  public:
    void commit(netCDFVID &n, size_t write_loc) override
    {
        const char *writable = char_rep.c_str();
        n.nc_put_vvar1_string(OGR_SGFS_Transaction::getVarId(), &write_loc,
                              &(writable));
    }

    unsigned long long count() override
    {
        return char_rep.size() + sizeof(*this);
    }  // account for actual character representation, this class

    nc_type getType() override
    {
        return NC_STRING;
    }

    void appendToLog(VSILFILE *f) override;

    OGR_SGFS_NC_String_Transaction(int i_varId, const char *pszVal)
        : char_rep(pszVal)
    {
        OGR_SGFS_Transaction::setVarId(i_varId);
    }
};

/* WTransactionLog
 * -
 * A temporary file which contains transactions to be written to a netCDF file.
 * Once the transaction log is created it is set on write mode, it can only be
 * read to after startRead() is called
 */
class WTransactionLog
{
    bool readMode = false;
    std::string wlogName;  // name of the temporary file, should be unique
    VSILFILE *log = nullptr;

    WTransactionLog(WTransactionLog &);  // avoid possible undefined behavior
    WTransactionLog operator=(const WTransactionLog &);

  public:
    bool logIsNull()
    {
        return log == nullptr;
    }
    void startLog();   // always call this first to open the file
    void startRead();  // then call this before reading it
    void push(MTPtr);

    // read mode
    MTPtr
    pop();  // to test for EOF, test to see if pointer returned is null ptr

    // construction, destruction
    explicit WTransactionLog(const std::string &logName);
    ~WTransactionLog();
};

/* OGR_NCScribe
 * Buffers several netCDF transactions in memory or in a log.
 * General scribe class
 */
class OGR_NCScribe
{
    netCDFVID &ncvd;
    WBuffer buf;
    WTransactionLog wl;
    bool singleDatumMode = false;

    std::queue<MTPtr> transactionQueue;
    std::map<int, size_t> varWriteInds;
    std::map<int, size_t> varMaxInds;

  public:
    /* size_t getWriteCount()
     * Return the total write count (happened + pending) of certain variable
     */
    size_t getWriteCount(int varId)
    {
        return this->varMaxInds.at(varId);
    }

    /* void commit_transaction()
     * Replays all transactions to disk (according to fs stipulations)
     */
    void commit_transaction();

    /* MTPtr pop()
     * Get the next transaction, if it exists.
     * If not, it will just return a shared_ptr with nullptr inside
     */
    MTPtr pop();

    /* void log_transacion()
     * Saves the current queued transactions to a log.
     */
    void log_transaction();

    /* void enqueue_transaction()
     * Add a transaction to perform
     * Once a transaction is enqueued, it will only be dequeued on commit
     */
    void enqueue_transaction(MTPtr transactionAdd);

    WBuffer &getMemBuffer()
    {
        return buf;
    }

    /* OGR_SGeometry_Field_Scribe()
     * Constructs a Field Scribe over a dataset
     */
    OGR_NCScribe(netCDFVID &ncd, const std::string &name) : ncvd(ncd), wl(name)
    {
    }

    /* setSingleDatumMode(...)
     * Enables or disables single datum mode
     * DO NOT use this when a commit is taking place, otherwise
     * corruption may occur...
     */
    void setSingleDatumMode(bool sdm)
    {
        this->singleDatumMode = sdm;
    }
};

class ncLayer_SG_Metadata
{
    int &ncID;  // ncid REF. which tracks ncID changes that may be made upstream

    netCDFVID &vDataset;
    OGR_NCScribe &ncb;
    geom_t writableType = NONE;
    std::string containerVarName;
    int containerVar_realID = INVALID_VAR_ID;
    bool interiorRingDetected =
        false;  // flips on when an interior ring polygon has been detected
    std::vector<int>
        node_coordinates_varIDs;  // ids in X, Y (and then possibly Z) order
    int node_coordinates_dimID = INVALID_DIM_ID;  // dim of all node_coordinates
    int node_count_dimID = INVALID_DIM_ID;        // node count dim
    int node_count_varID = INVALID_DIM_ID;
    int pnc_dimID =
        INVALID_DIM_ID;  // part node count dim AND interior ring dim
    int pnc_varID = INVALID_VAR_ID;
    int intring_varID = INVALID_VAR_ID;
    size_t next_write_pos_node_coord = 0;
    size_t next_write_pos_node_count = 0;
    size_t next_write_pos_pnc = 0;

  public:
    geom_t getWritableType()
    {
        return this->writableType;
    }
    void writeSGeometryFeature(SGeometry_Feature &ft);
    int get_containerRealID()
    {
        return this->containerVar_realID;
    }
    std::string get_containerName()
    {
        return this->containerVarName;
    }
    int get_node_count_dimID()
    {
        return this->node_count_dimID;
    }
    int get_node_coord_dimID()
    {
        return this->node_coordinates_dimID;
    }
    int get_pnc_dimID()
    {
        return this->pnc_dimID;
    }
    int get_pnc_varID()
    {
        return this->pnc_varID;
    }
    int get_intring_varID()
    {
        return this->intring_varID;
    }
    std::vector<int> &get_nodeCoordVarIDs()
    {
        return this->node_coordinates_varIDs;
    }
    size_t get_next_write_pos_node_coord()
    {
        return this->next_write_pos_node_coord;
    }
    size_t get_next_write_pos_node_count()
    {
        return this->next_write_pos_node_count;
    }
    size_t get_next_write_pos_pnc()
    {
        return this->next_write_pos_pnc;
    }
    bool getInteriorRingDetected()
    {
        return this->interiorRingDetected;
    }
    void initializeNewContainer(int containerVID);
    ncLayer_SG_Metadata(int &i_ncID, geom_t geo, netCDFVID &ncdf,
                        OGR_NCScribe &scribe);
};

/* WBufferManager
 * -
 * Simply takes a collection of buffers in and a quota limit and sums all the
 * usages up to establish if buffers are over the soft limit (collectively)
 *
 * The buffers added, however, are not memory managed by WBufferManager
 */
class WBufferManager
{
    unsigned long long buffer_soft_limit = 0;
    std::vector<WBuffer *> bufs;

  public:
    bool isOverQuota();
    void adjustLimit(unsigned long long lim)
    {
        this->buffer_soft_limit = lim;
    }
    void addBuffer(WBuffer *b)
    {
        this->bufs.push_back(b);
    }
    explicit WBufferManager(unsigned long long lim) : buffer_soft_limit(lim)
    {
    }
};

// Exception Classes
class SGWriter_Exception : public SG_Exception
{
  public:
    const char *get_err_msg() override
    {
        return "A general error occurred when writing a netCDF dataset";
    }
};

class SGWriter_Exception_NCWriteFailure : public SGWriter_Exception
{
    std::string msg;

  public:
    const char *get_err_msg() override
    {
        return this->msg.c_str();
    }
    SGWriter_Exception_NCWriteFailure(const char *layer_name,
                                      const char *failure_name,
                                      const char *failure_type);
};

class SGWriter_Exception_NCInqFailure : public SGWriter_Exception
{
    std::string msg;

  public:
    const char *get_err_msg() override
    {
        return this->msg.c_str();
    }
    SGWriter_Exception_NCInqFailure(const char *layer_name,
                                    const char *failure_name,
                                    const char *failure_type);
};

class SGWriter_Exception_NCDefFailure : public SGWriter_Exception
{
    std::string msg;

  public:
    const char *get_err_msg() override
    {
        return this->msg.c_str();
    }
    SGWriter_Exception_NCDefFailure(const char *layer_name,
                                    const char *failure_name,
                                    const char *failure_type);
};

class SGWriter_Exception_NullGeometry : public SGWriter_Exception
{
    std::string msg;

  public:
    const char *get_err_msg() override
    {
        return this->msg.c_str();
    }
    SGWriter_Exception_NullGeometry()
        : msg("A null  geometry was detected when writing a netCDF file. "
              "Empty geometries are not allowed.")
    {
    }
};

class SGWriter_Exception_NCDelFailure : public SGWriter_Exception
{
    std::string msg;

  public:
    const char *get_err_msg() override
    {
        return this->msg.c_str();
    }
    SGWriter_Exception_NCDelFailure(const char *layer, const char *what)
        : msg("[" + std::string(layer) +
              "] Failed to delete: " + std::string(what))
    {
    }
};

// Helper Functions that for writing

/* std::vector<..> writeGeometryContainer(...)
 * Writes a geometry container of a given geometry type given the following
 * arguments: int ncID - ncid as used in netcdf.h, group or file id std::string
 * name - what to name this container geom_t geometry_type - the geometry type
 * of the container std::vector<..> node_coordinate_names - variable names
 * corresponding to each axis Only writes attributes that are for sure required.
 * i.e. does NOT required interior ring for anything or part node count for
 * Polygons
 *
 * Returns: geometry container variable ID
 */
int write_Geometry_Container(
    int ncID, const std::string &name, geom_t geometry_type,
    const std::vector<std::string> &node_coordinate_names);

template <class W_type>
inline void NCWMapAllocIfNeeded(int varid, NCWMap &mapAdd, size_t numEntries,
                                std::vector<int> &v)
{
    if (mapAdd.count(varid) < 1)
    {
        mapAdd.insert(NCWEntry(varid, CPLMalloc(sizeof(W_type) * numEntries)));
        v.push_back(varid);
    }
}

template <class W_type>
inline void NCWMapWriteAndCommit(int varid, NCWMap &mapAdd, size_t currentEntry,
                                 size_t numEntries, W_type data,
                                 netCDFVID &vcdf)
{
    W_type *ptr = static_cast<W_type *>(mapAdd.at(varid));
    ptr[currentEntry] = data;
    static const size_t BEGIN = 0;

    // If all items are ready, write the array, and free it, delete the pointer
    if (currentEntry == (numEntries - 1))
    {
        try
        {
            // Write the whole array at once
            vcdf.nc_put_vvara_generic<W_type>(varid, &BEGIN, &numEntries, ptr);
        }
        catch (SG_Exception_VWrite_Failure &e)
        {
            CPLError(CE_Warning, CPLE_FileIO, "%s", e.get_err_msg());
        }

        CPLFree(mapAdd.at(varid));
        mapAdd.erase(varid);
    }
}
}  // namespace nccfdriver

#endif