Merge pull request #233 from ngageoint/compressedSiddByteProvider

Compressed sidd byte provider

six/modules/c++/samples/wscript

@@ -14,6 +14,7 @@ def build(bld):
                'test_create_sicd'                    : 'cli six.sicd sio.lite',
                'test_create_sicd_from_mem'           : 'cli six.sicd',
                'test_create_sidd_from_mem'           : 'cli six.sicd six.sidd',
+               'test_create_sidd_with_compressed_byte_provider' : 'cli six.sidd',
                'test_create_sidd_legend'             : 'cli six.sidd',
                'test_create_sidd'                    : 'cli six.sicd six.sidd sio.lite',
                'test_dump_images'                    : 'cli six.sicd six.sidd sio.lite',

six/modules/c++/six.sidd/include/six/sidd/CompressedSIDDByteProvider.h

@@ -0,0 +1,93 @@
+/* =========================================================================
+ * This file is part of six.sidd-c++
+ * =========================================================================
+ *
+ * (C) Copyright 2004 - 2018, MDA Information Systems LLC
+ *
+ * six.sidd-c++ is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; If not,
+ * see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __SIX_SIDD_COMPRESSED_SIDD_BYTE_PROVIDER_H__
+#define __SIX_SIDD_COMPRESSED_SIDD_BYTE_PROVIDER_H__
+
+#include <six/CompressedByteProvider.h>
+#include <six/sidd/DerivedData.h>
+
+namespace six
+{
+namespace sidd
+{
+/*!
+ * \class CompressedSIDDByteProvider
+ * \brief Used to provide corresponding compressed NITF bytes (including NITF headers)
+ * when provided with some AOI of the pixel data.  The idea is that if
+ * getBytes() is called multiple times, eventually for the entire image, the
+ * raw bytes provided back will be the entire NITF file.  This abstraction is
+ * useful if separate threads, processes, or even machines have only portions of
+ * the SIDD pixel data and are all trying to write out a single file; in that
+ * scenario, this class provides all the raw bytes corresponding to the caller's
+ * AOI, including NITF headers if necessary.  The caller does not need to
+ * understand anything about the NITF file layout in order to write out the
+ * file.  The bytes are intentionally provided back as a series of pointers
+ * rather than one contiguous block of memory in order to not perform any
+ * copies.  A single logical SIDD image which spans multiple NITF image segments
+ * is supported; unrelated SIDD images in one NITF are not yet supported.
+ */
+class CompressedSIDDByteProvider : public six::CompressedByteProvider
+{
+public:
+    /*!
+     * Constructor
+     *
+     * \param data Representation of the derived data
+     * \param schemPaths Directories or files of schema locations
+     * \param bytesPerBlock A vector for each image segment. Each inner vector
+     *        contains the compressed size for each block in the segment,
+     *        in bytes.
+     * \param numRowsPerBlock The number of rows per block.  Defaults to no
+     * blocking.
+     * \param numColsPerBlock The number of columns per block.  Defaults to no
+     * blocking.
+     * \param maxProductSize The max number of bytes in an image segment.
+     * By default this is set automatically for you based on NITF file rules.
+     */
+    CompressedSIDDByteProvider(const DerivedData& data,
+                               const std::vector<std::string>& schemaPaths,
+                               const std::vector<std::vector<size_t> >& bytesPerBlock,
+                               size_t numRowsPerBlock = 0,
+                               size_t numColsPerBlock = 0,
+                               size_t maxProductSize = 0);
+
+    /*!
+     * Constructor
+     * This option allows you to pass in an initialized writer,
+     * in case you need something specific in the header
+     *
+     * \param writer Initialized NITFWriteControl.  Must have all desired
+     * product size and blocking values set.
+     * \param schemaPaths Directories or files of schema locations
+     * \param bytesPerBlock A vector for each image segment. Each inner vector
+     *        contains the compressed size for each block in the segment,
+     *        in bytes.
+     */
+    CompressedSIDDByteProvider(const NITFWriteControl& writer,
+                               const std::vector<std::string>& schemaPaths,
+                               const std::vector<std::vector<size_t> >& bytesPerBlock);
+};
+}
+}
+
+#endif

six/modules/c++/six.sidd/source/CompressedSIDDByteProvider.cpp

@@ -0,0 +1,61 @@
+/* =========================================================================
+ * This file is part of six.sidd-c++
+ * =========================================================================
+ *
+ * (C) Copyright 2004 - 2018, MDA Information Systems LLC
+ *
+ * six.sidd-c++ is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; If not,
+ * see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <six/sidd/DerivedXMLControl.h>
+#include <six/sidd/CompressedSIDDByteProvider.h>
+
+namespace six
+{
+namespace sidd
+{
+CompressedSIDDByteProvider::CompressedSIDDByteProvider(
+        const DerivedData& data,
+        const std::vector<std::string>& schemaPaths,
+        const std::vector<std::vector<size_t> >& bytesPerBlock,
+        size_t numRowsPerBlock,
+        size_t numColsPerBlock,
+        size_t maxProductSize)
+{
+    XMLControlRegistry xmlRegistry;
+    xmlRegistry.addCreator(six::DataType::DERIVED,
+                           new six::XMLControlCreatorT<DerivedXMLControl>());
+
+    mem::SharedPtr<Container> container(new Container(
+            DataType::DERIVED));
+
+    // The container wants to take ownership of the data
+    // To avoid memory problems, we'll just clone it
+    container->addData(data.clone());
+
+    initialize(container, xmlRegistry, schemaPaths, bytesPerBlock,
+               maxProductSize, numRowsPerBlock, numColsPerBlock);
+}
+
+CompressedSIDDByteProvider::CompressedSIDDByteProvider(
+        const NITFWriteControl& writer,
+        const std::vector<std::string>& schemaPaths,
+        const std::vector<std::vector<size_t> >& bytesPerBlock)
+{
+    initialize(writer, schemaPaths, bytesPerBlock);
+}
+}
+}

six/modules/c++/six/include/six/ByteProvider.h

@@ -49,6 +49,22 @@ namespace six
  */
 class ByteProvider : public nitf::ByteProvider
 {
+public:
+    static void populateWriter(
+            mem::SharedPtr<Container> container,
+            const XMLControlRegistry& xmlRegistry,
+            size_t maxProductSize,
+            size_t numRowsPerBlock,
+            size_t numColsPerBlock,
+            NITFWriteControl& writer);
+
+    static void populateInitArgs(
+            const NITFWriteControl& writer,
+            const std::vector<std::string>& schemaPaths,
+            std::vector<std::string>& xmlStrings,
+            std::vector<PtrAndLength>& desData,
+            size_t& numRowsPerBlock,
+            size_t& numColsPerBlock);
 protected:
     /*!
      * Initialize the byte provider.  Must be called in the constructor of

six/modules/c++/six/include/six/CompressedByteProvider.h

@@ -0,0 +1,96 @@
+/* =========================================================================
+ * This file is part of six-c++
+ * =========================================================================
+ *
+ * (C) Copyright 2004 - 2018, MDA Information Systems LLC
+ *
+ * six-c++ is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; If not,
+ * see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __SIX_COMPRESSED_BYTE_PROVIDER_H__
+#define __SIX_COMPRESSED_BYTE_PROVIDER_H__
+
+#include <nitf/CompressedByteProvider.hpp>
+#include <mem/SharedPtr.h>
+#include <six/Container.h>
+#include <six/NITFWriteControl.h>
+#include <six/NITFSegmentInfo.h>
+#include <six/XMLControlFactory.h>
+
+namespace six
+{
+/*!
+ * \class CompressedByteProvider
+ * \brief Used to provide corresponding compressed NITF bytes
+ * (and uncompressed NITF headers) when provided with some AOI of the pixel data.
+ * The idea is that if getBytes() is called multiple times, eventually for the
+ * entire image, the raw bytes provided back will be the entire NITF file.
+ * This abstraction is useful if separate threads, processes, or even machines
+ * have only portions of the SICD/SIDD pixel data and are all trying to write
+ * out a single file; in that scenario, this class provides all the raw bytes
+ * corresponding to the caller's AOI, including NITF headers if necessary.
+ * The caller does not need to understand anything about the NITF file layout
+ * in order to write out the file.
+ * The bytes are intentionally provided back as a series of pointers
+ * rather than one contiguous block of memory in order to minimize the number of
+ * copies.
+ */
+class CompressedByteProvider : public nitf::CompressedByteProvider
+{
+protected:
+    /*!
+     * Initialize the byte provider.  Must be called in the constructor of
+     * inheriting classes.
+     *
+     * \param container Container initialized with all associated data
+     * \param xmlRegistry XML registry
+     * \param schemaPaths Directories or files of schema locations
+     * \param bytesPerBlock A vector for each image segment. Each inner vector
+     *        contains the compressed size for each block in the segment,
+     *        in bytes.
+     * \param maxProductSize The max number of bytes in an image segment.
+     * \param numRowsPerBlock The number of rows per block.  Only applies for
+     * SIDD.  Defaults to no blocking.
+     * \param numColsPerBlock The number of columns per block.  Only applies
+     * for SIDD.  Defaults to no blocking.
+     */
+    void initialize(mem::SharedPtr<Container> container,
+                    const XMLControlRegistry& xmlRegistry,
+                    const std::vector<std::string>& schemaPaths,
+                    const std::vector<std::vector<size_t> >& bytesPerBlock,
+                    size_t maxProductSize,
+                    size_t numRowsPerBlock = 0,
+                    size_t numColsPerBlock = 0);
+
+    /*!
+     * Initialize the byte provider.  Must be called in the constructor of
+     * inheriting classes.
+     *
+     * \param writer Writer.  Must be initialized via
+     * NITFWriteControl::initialize() and have all desired product size and
+     * blocking values set.
+     * \param schemaPaths Directories or files of schema locations
+     * \param bytesPerBlock A vector for each image segment. Each inner vector
+     *        contains the compressed size for each block in the segment,
+     *        in bytes.
+     */
+    void initialize(const NITFWriteControl& writer,
+                    const std::vector<std::string>& schemaPaths,
+                    const std::vector<std::vector<size_t> >& bytesPerBlock);
+};
+}
+
+#endif

six/modules/c++/six/source/ByteProvider.cpp

@@ -26,12 +26,14 @@
 
 namespace six
 {
-void ByteProvider::initialize(mem::SharedPtr<Container> container,
-                              const XMLControlRegistry& xmlRegistry,
-                              const std::vector<std::string>& schemaPaths,
-                              size_t maxProductSize,
-                              size_t numRowsPerBlock,
-                              size_t numColsPerBlock)
+
+void ByteProvider::populateWriter(
+        mem::SharedPtr<Container> container,
+        const XMLControlRegistry& xmlRegistry,
+        size_t maxProductSize,
+        size_t numRowsPerBlock,
+        size_t numColsPerBlock,
+        NITFWriteControl& writer)
 {
     if (container->getNumData() != 1)
     {
@@ -42,7 +44,6 @@ void ByteProvider::initialize(mem::SharedPtr<Container> container,
 
     const six::Data* const data = container->getData(0);
 
-    NITFWriteControl writer;
     writer.setXMLControlRegistry(&xmlRegistry);
 
     if (maxProductSize != 0)
@@ -69,12 +70,15 @@ void ByteProvider::initialize(mem::SharedPtr<Container> container,
     }
 
     writer.initialize(container);
-
-    initialize(writer, schemaPaths);
 }
 
-void ByteProvider::initialize(const NITFWriteControl& writer,
-                              const std::vector<std::string>& schemaPaths)
+void ByteProvider::populateInitArgs(
+        const NITFWriteControl& writer,
+        const std::vector<std::string>& schemaPaths,
+        std::vector<std::string>& xmlStrings,
+        std::vector<PtrAndLength>& desData,
+        size_t& numRowsPerBlock,
+        size_t& numColsPerBlock)
 {
     // Sanity check the container
     mem::SharedPtr<const Container> container = writer.getContainer();
@@ -111,8 +115,8 @@ void ByteProvider::initialize(const NITFWriteControl& writer,
     // This memory must stay around until the call to the
     // base class's initialize() method
     logging::NullLogger logger;
-    std::vector<std::string> xmlStrings(container->getNumData());
-    std::vector<PtrAndLength> desData(xmlStrings.size());
+    xmlStrings.resize(container->getNumData());
+    desData.resize(xmlStrings.size());
     for (size_t ii = 0; ii < xmlStrings.size(); ++ii)
     {
         std::string& xmlString(xmlStrings[ii]);
@@ -128,13 +132,43 @@ void ByteProvider::initialize(const NITFWriteControl& writer,
     const Options& options(writer.getOptions());
     const Parameter zero(0);
 
-    const size_t numRowsPerBlock = static_cast<sys::Uint32_T>(
+    numRowsPerBlock = static_cast<sys::Uint32_T>(
             options.getParameter(NITFWriteControl::OPT_NUM_ROWS_PER_BLOCK,
                                  zero));
 
-    const size_t numColsPerBlock = static_cast<sys::Uint32_T>(
+    numColsPerBlock = static_cast<sys::Uint32_T>(
             options.getParameter(NITFWriteControl::OPT_NUM_COLS_PER_BLOCK,
                                  zero));
+}
+
+void ByteProvider::initialize(mem::SharedPtr<Container> container,
+                              const XMLControlRegistry& xmlRegistry,
+                              const std::vector<std::string>& schemaPaths,
+                              size_t maxProductSize,
+                              size_t numRowsPerBlock,
+                              size_t numColsPerBlock)
+{
+    NITFWriteControl writer;
+    populateWriter(container, xmlRegistry, maxProductSize, numRowsPerBlock,
+            numColsPerBlock, writer);
+    initialize(writer, schemaPaths);
+}
+
+void ByteProvider::initialize(const NITFWriteControl& writer,
+                              const std::vector<std::string>& schemaPaths)
+{
+    // We don't explicitly use it, but each element in desData has a pointer
+    // into this vector, so we need it to stick around
+    std::vector<std::string> xmlStrings;
+    std::vector<PtrAndLength> desData;
+    size_t numRowsPerBlock;
+    size_t numColsPerBlock;
+    populateInitArgs(writer,
+                     schemaPaths,
+                     xmlStrings,
+                     desData,
+                     numRowsPerBlock,
+                     numColsPerBlock);
 
     // Do the full initialization
     nitf::Record record = writer.getRecord();