Blame IlmImfTest/testDeepScanLineHuge.cpp

Packit 0d464f
///////////////////////////////////////////////////////////////////////////
Packit 0d464f
//
Packit 0d464f
// Copyright (c) 2011, Industrial Light & Magic, a division of Lucas
Packit 0d464f
// Digital Ltd. LLC
Packit 0d464f
//
Packit 0d464f
// All rights reserved.
Packit 0d464f
//
Packit 0d464f
// Redistribution and use in source and binary forms, with or without
Packit 0d464f
// modification, are permitted provided that the following conditions are
Packit 0d464f
// met:
Packit 0d464f
// *       Redistributions of source code must retain the above copyright
Packit 0d464f
// notice, this list of conditions and the following disclaimer.
Packit 0d464f
// *       Redistributions in binary form must reproduce the above
Packit 0d464f
// copyright notice, this list of conditions and the following disclaimer
Packit 0d464f
// in the documentation and/or other materials provided with the
Packit 0d464f
// distribution.
Packit 0d464f
// *       Neither the name of Industrial Light & Magic nor the names of
Packit 0d464f
// its contributors may be used to endorse or promote products derived
Packit 0d464f
// from this software without specific prior written permission.
Packit 0d464f
//
Packit 0d464f
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
Packit 0d464f
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
Packit 0d464f
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
Packit 0d464f
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
Packit 0d464f
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
Packit 0d464f
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
Packit 0d464f
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
Packit 0d464f
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
Packit 0d464f
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
Packit 0d464f
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
Packit 0d464f
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Packit 0d464f
//
Packit 0d464f
///////////////////////////////////////////////////////////////////////////
Packit 0d464f
Packit 0d464f
#include "testDeepScanLineBasic.h"
Packit 0d464f
Packit 0d464f
#include "ImfDeepScanLineInputFile.h"
Packit 0d464f
#include "ImfDeepScanLineOutputFile.h"
Packit 0d464f
#include "ImfDeepFrameBuffer.h"
Packit 0d464f
#include "ImfPartType.h"
Packit 0d464f
#include "ImfChannelList.h"
Packit 0d464f
#include "ImfArray.h"
Packit 0d464f
#include "IlmThreadPool.h"
Packit 0d464f
Packit 0d464f
#include "tmpDir.h"
Packit 0d464f
Packit 0d464f
#include <assert.h>
Packit 0d464f
#include <string.h>
Packit 0d464f
#include <stdlib.h>
Packit 0d464f
#include <stdio.h>
Packit 0d464f
Packit 0d464f
#include <vector>
Packit 0d464f
Packit 0d464f
namespace IMF = OPENEXR_IMF_NAMESPACE;
Packit 0d464f
using namespace IMF;
Packit 0d464f
using namespace std;
Packit 0d464f
using namespace IMATH_NAMESPACE;
Packit 0d464f
using namespace ILMTHREAD_NAMESPACE;
Packit 0d464f
Packit 0d464f
namespace
Packit 0d464f
{
Packit 0d464f
Packit 0d464f
const int width = 8193;
Packit 0d464f
const int height = 1;
Packit 0d464f
const int minX = 0;
Packit 0d464f
const int minY = 0;
Packit 0d464f
const long numGib = 1; // number of GiB to allocate for huge test
Packit 0d464f
const Box2i dataWindow(V2i(minX, minY), V2i(minX + width - 1, minY + height - 1));
Packit 0d464f
const Box2i displayWindow(V2i(0, 0), V2i(minX + width * 2, minY + height * 2));
Packit 0d464f
Packit 0d464f
vector<int> channelTypes;
Packit 0d464f
Array2D<unsigned int> sampleCount;
Packit 0d464f
vector<unsigned char> storage; // actual pixel storage for entire image (effectively)
Packit 0d464f
Packit 0d464f
Header header;
Packit 0d464f
Packit 0d464f
void
Packit 0d464f
generateRandomFile (int channelCount,
Packit 0d464f
                    Compression compression,
Packit 0d464f
                    bool random_channel_data,
Packit 0d464f
                    const std::string & fn)
Packit 0d464f
{
Packit 0d464f
    cout << "generating ... " << flush;
Packit 0d464f
    header = Header(displayWindow, dataWindow,
Packit 0d464f
                    1,
Packit 0d464f
                    IMATH_NAMESPACE::V2f (0, 0),
Packit 0d464f
                    1,
Packit 0d464f
                    INCREASING_Y,
Packit 0d464f
                    compression);
Packit 0d464f
Packit 0d464f
    cout << "compression " << compression << " " << flush;
Packit 0d464f
Packit 0d464f
    //
Packit 0d464f
    // Add channels.
Packit 0d464f
    //
Packit 0d464f
Packit 0d464f
    channelTypes.clear();
Packit 0d464f
Packit 0d464f
    for (int i = 0; i < channelCount; i++)
Packit 0d464f
    {
Packit 0d464f
        int type = rand() % 3;
Packit 0d464f
        stringstream ss;
Packit 0d464f
        ss << i;
Packit 0d464f
        string str = ss.str();
Packit 0d464f
        if (type == 0)
Packit 0d464f
            header.channels().insert(str, Channel(IMF::UINT));
Packit 0d464f
        if (type == 1)
Packit 0d464f
            header.channels().insert(str, Channel(IMF::HALF));
Packit 0d464f
        if (type == 2)
Packit 0d464f
            header.channels().insert(str, Channel(IMF::FLOAT));
Packit 0d464f
        channelTypes.push_back(type);
Packit 0d464f
    }
Packit 0d464f
Packit 0d464f
    header.setType(DEEPSCANLINE);
Packit 0d464f
Packit 0d464f
    Array<Array2D< void* > > data(channelCount);
Packit 0d464f
    for (int i = 0; i < channelCount; i++)
Packit 0d464f
        data[i].resizeErase(height, width);
Packit 0d464f
Packit 0d464f
    sampleCount.resizeErase(height, width);
Packit 0d464f
Packit 0d464f
    remove (fn.c_str());
Packit 0d464f
    DeepScanLineOutputFile file (fn.c_str(), header, 8);
Packit 0d464f
Packit 0d464f
    DeepFrameBuffer frameBuffer;
Packit 0d464f
Packit 0d464f
    frameBuffer.insertSampleCountSlice (Slice (IMF::UINT,                    // type // 7
Packit 0d464f
                                        (char *) (&sampleCount[0][0]
Packit 0d464f
                                                  - dataWindow.min.x
Packit 0d464f
                                                  - dataWindow.min.y * width),               // base // 8
Packit 0d464f
                                        sizeof (unsigned int) * 1,          // xStride// 9
Packit 0d464f
                                        sizeof (unsigned int) * width));    // yStride// 10
Packit 0d464f
Packit 0d464f
    
Packit 0d464f
    // count total size of all pixels
Packit 0d464f
    Int64 bytes_per_sample = 0;
Packit 0d464f
    for (int i = 0; i < channelCount; i++)
Packit 0d464f
    {
Packit 0d464f
        PixelType type;
Packit 0d464f
        if (channelTypes[i] == 0)
Packit 0d464f
            type = IMF::UINT;
Packit 0d464f
        if (channelTypes[i] == 1)
Packit 0d464f
            type = IMF::HALF;
Packit 0d464f
        if (channelTypes[i] == 2)
Packit 0d464f
            type = IMF::FLOAT;
Packit 0d464f
Packit 0d464f
        stringstream ss;
Packit 0d464f
        ss << i;
Packit 0d464f
        string str = ss.str();
Packit 0d464f
Packit 0d464f
        int sampleSize;
Packit 0d464f
        if (channelTypes[i] == 0) sampleSize = sizeof (unsigned int);
Packit 0d464f
        if (channelTypes[i] == 1) sampleSize = sizeof (half);
Packit 0d464f
        if (channelTypes[i] == 2) sampleSize = sizeof (float);
Packit 0d464f
Packit 0d464f
        int pointerSize = sizeof(char *);
Packit 0d464f
Packit 0d464f
         bytes_per_sample+=sampleSize;
Packit 0d464f
        
Packit 0d464f
        frameBuffer.insert (str,                            // name // 6
Packit 0d464f
                            DeepSlice (type,                    // type // 7
Packit 0d464f
                            (char *) (&data[i][0][0]
Packit 0d464f
                                      - dataWindow.min.x
Packit 0d464f
                                      - dataWindow.min.y * width),               // base // 8
Packit 0d464f
                            pointerSize * 1,          // xStride// 9
Packit 0d464f
                            pointerSize * width,      // yStride// 10
Packit 0d464f
                            sampleSize));             // sampleStride
Packit 0d464f
    }
Packit 0d464f
Packit 0d464f
    file.setFrameBuffer(frameBuffer);
Packit 0d464f
Packit 0d464f
    cout << "writing file " << endl;
Packit 0d464f
Packit 0d464f
    Int64 total_number_of_samples = 0;
Packit 0d464f
    
Packit 0d464f
    // compute ideal number of samples per pixel assuming we want abotut 5GiB of data
Packit 0d464f
    // int samples_per_pixel = int(5l*1024l*1024l*1024l/Int64(width*height)) / bytes_per_sample;
Packit 0d464f
Packit 0d464f
    // compute ideal number of samples per pixel assuming we want abotut 15GiB of data
Packit 0d464f
    int samples_per_pixel = int(numGib*1024l*1024l*1024l/Int64(width*height)) / bytes_per_sample;
Packit 0d464f
    
Packit 0d464f
    cout << "  generating approx. " << samples_per_pixel << " samples per pixel\n";
Packit 0d464f
    
Packit 0d464f
    for (int i = 0; i < height; i++)
Packit 0d464f
    {
Packit 0d464f
            for (int j = 0; j < width; j++)
Packit 0d464f
            {
Packit 0d464f
                sampleCount[i][j] = (rand() % 4000) + (samples_per_pixel-2000);
Packit 0d464f
                total_number_of_samples += sampleCount[i][j];
Packit 0d464f
            }
Packit 0d464f
    }
Packit 0d464f
    
Packit 0d464f
    cout << "  total number of samples: " << total_number_of_samples << std::endl;
Packit 0d464f
    cout << "  storage required: " << total_number_of_samples*bytes_per_sample << " bytes (" <<
Packit 0d464f
    ((total_number_of_samples*bytes_per_sample)>>30) << "GiB)" <<  std::endl;
Packit 0d464f
    
Packit 0d464f
    
Packit 0d464f
    //
Packit 0d464f
    // storage layout scheme:
Packit 0d464f
    // [Pixel1: [Channel1: [Sample1 Sample2 Sample...] ] [Channel2: [Sample1 Sample2...] ] [Channnel...] ]
Packit 0d464f
    // [Pixel2: [Channel1: [Sample1 Sample2 Sample...] ] [Channel2: [Sample1 Sample2...] ] [Channnel...] ]
Packit 0d464f
    // [Pixel...]
Packit 0d464f
    //
Packit 0d464f
    storage.resize(total_number_of_samples*bytes_per_sample);
Packit 0d464f
    
Packit 0d464f
Packit 0d464f
    
Packit 0d464f
    Int64 write_pointer=0;
Packit 0d464f
    
Packit 0d464f
    for (int i = 0; i < height; i++)
Packit 0d464f
    {
Packit 0d464f
        //
Packit 0d464f
        // Fill in data at the last minute.
Packit 0d464f
        //
Packit 0d464f
        
Packit 0d464f
        
Packit 0d464f
        for (int j = 0; j < width; j++)
Packit 0d464f
        {
Packit 0d464f
                for (int k = 0; k < channelCount; k++)
Packit 0d464f
                {
Packit 0d464f
                    data[k][i][j]=&storage[write_pointer];
Packit 0d464f
                    if (channelTypes[k] == 0)
Packit 0d464f
                        write_pointer+=sizeof(int)*sampleCount[i][j];
Packit 0d464f
                    if (channelTypes[k] == 1)
Packit 0d464f
                        write_pointer+=sizeof(half)*sampleCount[i][j];
Packit 0d464f
                    if (channelTypes[k] == 2)
Packit 0d464f
                        write_pointer+=sizeof(float)*sampleCount[i][j];
Packit 0d464f
                    
Packit 0d464f
                    if(random_channel_data)
Packit 0d464f
                    {
Packit 0d464f
                        for (int l = 0; l < sampleCount[i][j]; l++)
Packit 0d464f
                        {
Packit 0d464f
                            if (channelTypes[k] == 0)
Packit 0d464f
                                ((unsigned int*)data[k][i][j])[l] = rand();
Packit 0d464f
                            if (channelTypes[k] == 1)
Packit 0d464f
                                ((half*)data[k][i][j])[l] = rand()/RAND_MAX;
Packit 0d464f
                            if (channelTypes[k] == 2)
Packit 0d464f
                                ((float*)data[k][i][j])[l] = rand()/RAND_MAX;
Packit 0d464f
                        }
Packit 0d464f
                    }
Packit 0d464f
                    else
Packit 0d464f
                    {
Packit 0d464f
                      for (int l = 0; l < sampleCount[i][j]; l++)
Packit 0d464f
                      {
Packit 0d464f
                          if (channelTypes[k] == 0)
Packit 0d464f
                              ((unsigned int*)data[k][i][j])[l] = (i * width + j) % 2049;
Packit 0d464f
                          if (channelTypes[k] == 1)
Packit 0d464f
                              ((half*)data[k][i][j])[l] = (i * width + j) % 2049;
Packit 0d464f
                          if (channelTypes[k] == 2)
Packit 0d464f
                              ((float*)data[k][i][j])[l] = (i * width + j) % 2049;
Packit 0d464f
                      }
Packit 0d464f
                   }
Packit 0d464f
                }
Packit 0d464f
        
Packit 0d464f
        }
Packit 0d464f
    }
Packit 0d464f
Packit 0d464f
    cout << " data prepared, writing ...";
Packit 0d464f
Packit 0d464f
    file.writePixels(height);
Packit 0d464f
    cout << " data written\n";
Packit 0d464f
    
Packit 0d464f
}
Packit 0d464f
Packit 0d464f
void
Packit 0d464f
readFile (int channelCount, bool bulkRead, const std::string & fn)
Packit 0d464f
{
Packit 0d464f
    cout << "reading \n" << flush;
Packit 0d464f
Packit 0d464f
    DeepScanLineInputFile file (fn.c_str(), 8);
Packit 0d464f
Packit 0d464f
    const Header& fileHeader = file.header();
Packit 0d464f
    assert (fileHeader.displayWindow() == header.displayWindow());
Packit 0d464f
    assert (fileHeader.dataWindow() == header.dataWindow());
Packit 0d464f
    assert (fileHeader.pixelAspectRatio() == header.pixelAspectRatio());
Packit 0d464f
    assert (fileHeader.screenWindowCenter() == header.screenWindowCenter());
Packit 0d464f
    assert (fileHeader.screenWindowWidth() == header.screenWindowWidth());
Packit 0d464f
    assert (fileHeader.lineOrder() == header.lineOrder());
Packit 0d464f
    assert (fileHeader.compression() == header.compression());
Packit 0d464f
    assert (fileHeader.channels() == header.channels());
Packit 0d464f
    assert (fileHeader.type() == header.type());
Packit 0d464f
Packit 0d464f
    Array2D<unsigned int> localSampleCount;
Packit 0d464f
    localSampleCount.resizeErase(height, width);
Packit 0d464f
    Array<Array2D< void* > > data(channelCount);
Packit 0d464f
    for (int i = 0; i < channelCount; i++)
Packit 0d464f
        data[i].resizeErase(height, width);
Packit 0d464f
    
Packit 0d464f
    DeepFrameBuffer frameBuffer;
Packit 0d464f
Packit 0d464f
    frameBuffer.insertSampleCountSlice (Slice (IMF::UINT,                    // type // 7
Packit 0d464f
                                        (char *) (&localSampleCount[0][0]
Packit 0d464f
                                                  - dataWindow.min.x
Packit 0d464f
                                                  - dataWindow.min.y * width),               // base // 8)
Packit 0d464f
                                        sizeof (unsigned int) * 1,          // xStride// 9
Packit 0d464f
                                        sizeof (unsigned int) * width));    // yStride// 10
Packit 0d464f
Packit 0d464f
    Int64 bytes_per_sample=0;
Packit 0d464f
    
Packit 0d464f
    for (int i = 0; i < channelCount; i++)
Packit 0d464f
    {
Packit 0d464f
        PixelType type;
Packit 0d464f
        if (channelTypes[i] == 0)
Packit 0d464f
            type = IMF::UINT;
Packit 0d464f
        if (channelTypes[i] == 1)
Packit 0d464f
            type = IMF::HALF;
Packit 0d464f
        if (channelTypes[i] == 2)
Packit 0d464f
            type = IMF::FLOAT;
Packit 0d464f
Packit 0d464f
        stringstream ss;
Packit 0d464f
        ss << i;
Packit 0d464f
        string str = ss.str();
Packit 0d464f
Packit 0d464f
        int sampleSize;
Packit 0d464f
        if (channelTypes[i] == 0) sampleSize = sizeof (unsigned int);
Packit 0d464f
        if (channelTypes[i] == 1) sampleSize = sizeof (half);
Packit 0d464f
        if (channelTypes[i] == 2) sampleSize = sizeof (float);
Packit 0d464f
Packit 0d464f
        int pointerSize = sizeof (char *);
Packit 0d464f
Packit 0d464f
        bytes_per_sample+=sampleSize;
Packit 0d464f
        
Packit 0d464f
        frameBuffer.insert (str,                            // name // 6
Packit 0d464f
                            DeepSlice (type,                    // type // 7
Packit 0d464f
                            (char *) (&data[i][0][0]
Packit 0d464f
                                      - dataWindow.min.x
Packit 0d464f
                                      - dataWindow.min.y * width),               // base // 8)
Packit 0d464f
                            pointerSize * 1,          // xStride// 9
Packit 0d464f
                            pointerSize * width,      // yStride// 10
Packit 0d464f
                            sampleSize));             // sampleStride
Packit 0d464f
    }
Packit 0d464f
Packit 0d464f
    file.setFrameBuffer(frameBuffer);
Packit 0d464f
Packit 0d464f
    file.readPixelSampleCounts(dataWindow.min.y, dataWindow.max.y);
Packit 0d464f
    Int64 total_pixel_count = 0;
Packit 0d464f
    
Packit 0d464f
    for (int i = 0; i < dataWindow.max.y - dataWindow.min.y + 1; i++)
Packit 0d464f
    {
Packit 0d464f
         int y = i + dataWindow.min.y;
Packit 0d464f
Packit 0d464f
         
Packit 0d464f
         for (int j = 0; j < width; j++)
Packit 0d464f
         {
Packit 0d464f
              assert(localSampleCount[i][j] == sampleCount[i][j]);
Packit 0d464f
              total_pixel_count += localSampleCount[i][j];
Packit 0d464f
         }     
Packit 0d464f
    }
Packit 0d464f
    
Packit 0d464f
    
Packit 0d464f
    vector<char> localstorage(total_pixel_count*bytes_per_sample);
Packit 0d464f
    
Packit 0d464f
    Int64 write_pointer=0;
Packit 0d464f
    
Packit 0d464f
    for (int i = 0; i < height; i++)
Packit 0d464f
    {
Packit 0d464f
        //
Packit 0d464f
        // Fill in data at the last minute.
Packit 0d464f
        //
Packit 0d464f
        
Packit 0d464f
        
Packit 0d464f
        for (int j = 0; j < width; j++)
Packit 0d464f
        {
Packit 0d464f
            for (int k = 0; k < channelCount; k++)
Packit 0d464f
            {
Packit 0d464f
                data[k][i][j]=&localstorage[write_pointer];
Packit 0d464f
                if (channelTypes[k] == 0)
Packit 0d464f
                    write_pointer+=sizeof(int)*sampleCount[i][j];
Packit 0d464f
                if (channelTypes[k] == 1)
Packit 0d464f
                    write_pointer+=sizeof(half)*sampleCount[i][j];
Packit 0d464f
                if (channelTypes[k] == 2)
Packit 0d464f
                    write_pointer+=sizeof(float)*sampleCount[i][j];
Packit 0d464f
            }
Packit 0d464f
        }
Packit 0d464f
    }
Packit 0d464f
Packit 0d464f
    cout << "reading image data ... " << flush;
Packit 0d464f
    
Packit 0d464f
    file.readPixels(dataWindow.min.y, dataWindow.max.y);
Packit 0d464f
Packit 0d464f
    cout << " image read \n" << flush;
Packit 0d464f
    
Packit 0d464f
}
Packit 0d464f
Packit 0d464f
void
Packit 0d464f
readWriteTest (int channelCount,
Packit 0d464f
               int testTimes,
Packit 0d464f
               bool random_channel_data,
Packit 0d464f
               const std::string & fn)
Packit 0d464f
{
Packit 0d464f
    cout << "Testing files with " << channelCount << " channels " << testTimes << " times."
Packit 0d464f
         << endl << flush;
Packit 0d464f
    for (int i = 0; i < testTimes; i++)
Packit 0d464f
    {
Packit 0d464f
        int compressionIndex = i % 3;
Packit 0d464f
        Compression compression;
Packit 0d464f
        switch (compressionIndex)
Packit 0d464f
        {
Packit 0d464f
            case 0:
Packit 0d464f
                compression = NO_COMPRESSION;
Packit 0d464f
                break;
Packit 0d464f
            case 1:
Packit 0d464f
                compression = RLE_COMPRESSION;
Packit 0d464f
                break;
Packit 0d464f
            case 2:
Packit 0d464f
                compression = ZIPS_COMPRESSION;
Packit 0d464f
                break;
Packit 0d464f
        }
Packit 0d464f
Packit 0d464f
        generateRandomFile (channelCount, compression, random_channel_data, fn);
Packit 0d464f
        readFile (channelCount, false, fn);
Packit 0d464f
        remove (fn.c_str());
Packit 0d464f
     
Packit 0d464f
    }
Packit 0d464f
}
Packit 0d464f
Packit 0d464f
}; // namespace
Packit 0d464f
Packit 0d464f
void testDeepScanLineHuge (const std::string & tempDir)
Packit 0d464f
{
Packit 0d464f
    try
Packit 0d464f
    {
Packit 0d464f
        cout << "\n\nTesting the DeepScanLineInput/OutputFile for huge scanlines:\n" << endl;
Packit 0d464f
Packit 0d464f
        srand(1);
Packit 0d464f
        std::string fn = tempDir + "imf_test_deep_scanline_huge.exr";
Packit 0d464f
Packit 0d464f
        readWriteTest (10, 5 , false, fn);
Packit 0d464f
        readWriteTest (10, 5 , true,  fn);
Packit 0d464f
        cout << "ok\n" << endl;
Packit 0d464f
    }
Packit 0d464f
    catch (const std::exception &e)
Packit 0d464f
    {
Packit 0d464f
        cerr << "ERROR -- caught exception: " << e.what() << endl;
Packit 0d464f
        assert (false);
Packit 0d464f
    }
Packit 0d464f
}
Packit 0d464f