Merge pull request henesissrl#1 from henesissrl/section_load

Add feature to read only a slice of a dataset in C++.

.gitignore

@@ -1 +1,2 @@
-data/*
+data/*
+build/*

erg/erg.cpp

@@ -320,14 +320,16 @@ size_t Reader::readAll(std::vector<uint8_t*>& values, const std::vector<size_t>&
     return readRows;
 }
 
-size_t Reader::read(const size_t qindex, uint8_t* outData, const size_t size)
+size_t Reader::read(const size_t qindex, uint8_t* dst, const size_t size)
 {
+    return read(qindex, 0, mRecordsCount, dst, size);
+    /*
     if(qindex>=mQuantities.size())
         throw std::runtime_error("Index "+std::to_string(qindex)+" is out of bounds.");
 
     const Quantity& qt = mQuantities[qindex];
 
-    if((qt.size*mRecordsCount)<size)
+    if((qt.size*mRecordsCount)>size)
         throw std::runtime_error("Not enough data allocated");
 
     memset(outData, 0, size);
@@ -356,6 +358,49 @@ size_t Reader::read(const size_t qindex, uint8_t* outData, const size_t size)
         arrayBe2Host(outData, qt.size, readRows);
 
     return readRows;
+    */
+}
+
+size_t Reader::read(const size_t qindex, const size_t from, const size_t count, uint8_t* dst, const size_t size)
+{
+    if(qindex>=mQuantities.size())
+        throw std::runtime_error("Index "+std::to_string(qindex)+" is out of bounds.");
+
+    const Quantity& qt = mQuantities[qindex];
+
+    size_t expectedSize = qt.size * count;
+    if(expectedSize>size)
+        throw std::runtime_error("Not enough data allocated: "+std::to_string(size)+\
+                                 " instead of "+std::to_string(expectedSize)+" bytes.");
+
+    memset(dst, 0, size);
+    const size_t inOffset = qt.offset;
+
+    // Skip initial bytes
+    size_t bytesToSkip = initialSkipBytes() + mRecordSize * from;
+    mFile.seekg(bytesToSkip, std::ios_base::beg);
+
+    std::vector<uint8_t> row(mRecordSize, 0);
+    uint8_t* rowData = row.data();
+    size_t readRows = 0;
+    for(size_t index=0; index<count; ++index)
+    {
+        mFile.read(reinterpret_cast<char*>(rowData), mRecordSize);
+        if(mFile.eof())
+            break;
+
+        readRows += 1;
+
+        const size_t outOffset = index * qt.size;
+        std::memcpy(dst + outOffset,  rowData + inOffset, qt.size);
+    }
+
+    if(mByteOrder==ByteOrder::LittelEndian)
+        arrayLe2Host(dst, qt.size, readRows);
+    else
+        arrayBe2Host(dst, qt.size, readRows);
+
+    return readRows;
 }
 
 size_t Reader::index(const std::string &qname) const noexcept(false)

erg/erg.h

@@ -147,7 +147,7 @@ class Quantity
     std::string unit;   //!< The unit. It can be an empty string.
     std::string typeStr; //!< The type represented as string.
     Type type;          //!< The type.
-    size_t size;        //!< The size in bytes.
+    size_t size;        //!< The size in bytes of each data element.
     size_t offset;      //!< The offset in bytes from the start of the record.
 };
 
@@ -239,11 +239,23 @@ class Reader
      * \brief Read a single dataset from the file
      *
      * \param qindex Index of the dataset to read
-     * \param outData The pre-allocated destination memory
+     * \param dst The pre-allocated destination memory
      * \param size The size of the allocated memory
      * \return The number of records that has been read.
      */
-    size_t read(const size_t qindex, uint8_t* outData, const size_t size);
+    size_t read(const size_t qindex, uint8_t* dst, const size_t size);
+
+    /*!
+     * \brief Read a slice of single dataset from the file
+     *
+     * \param qindex Index of the dataset to read
+     * \param from Index of the first record to read
+     * \param count Maximum number of records to read
+     * \param dst The pre-allocated destination memory
+     * \param size The size of the allocated memory
+     * \return The number of records that has been read.
+     */
+    size_t read(const size_t qindex, const size_t from, const size_t count, uint8_t* dst, const size_t size);
 
     /*!
      * \brief Size in bytes of the dataset at the current index.

test/main.cpp

@@ -269,14 +269,24 @@ TEST(Reader, Read)
 
     std::vector<double> Time(parser.records(), 0.0);
     size_t timeIndex = parser.index("Data_8");
-    size_t numRows = parser.read(timeIndex, reinterpret_cast<uint8_t*>(Time.data()), Time.size());
+    size_t numRows = parser.read(timeIndex, reinterpret_cast<uint8_t*>(Time.data()), Time.size()*sizeof(double));
     ASSERT_EQ(numRows, parser.records());
 
     for(int i=0; i<numRows; ++i)
     {
         int value = std::lround(Time[i]*100.0);
         ASSERT_EQ(value, i);
     }
+
+    std::vector<double> Time2(100, 0.0);
+    numRows = parser.read(timeIndex, 1000, 100, reinterpret_cast<uint8_t*>(Time2.data()), Time2.size()*sizeof(double));
+    ASSERT_EQ(numRows, 100);
+
+    for(int i=0; i<numRows; ++i)
+    {
+        int value = std::lround(Time2[i]*100.0);
+        ASSERT_EQ(value, i+1000);
+    }
 }
 
 int main(int argc, char **argv) {