///////////////////////////////////////////////////////////////////////////

//

// Copyright (c) 2011, Industrial Light & Magic, a division of Lucas

// Digital Ltd. LLC

//

// All rights reserved.

//

// Redistribution and use in source and binary forms, with or without

// modification, are permitted provided that the following conditions are

// met:

// *       Redistributions of source code must retain the above copyright

// notice, this list of conditions and the following disclaimer.

// *       Redistributions in binary form must reproduce the above

// copyright notice, this list of conditions and the following disclaimer

// in the documentation and/or other materials provided with the

// distribution.

// *       Neither the name of Industrial Light & Magic nor the names of

// its contributors may be used to endorse or promote products derived

// from this software without specific prior written permission.

//

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

//

///////////////////////////////////////////////////////////////////////////

//-----------------------------------------------------------------------------

//

//      class DeepTiledOutputFile

//

//-----------------------------------------------------------------------------

#include "ImfDeepTiledOutputFile.h"

#include "ImfDeepTiledInputFile.h"

#include "ImfDeepTiledInputPart.h"

#include "ImfInputFile.h"

#include "ImfTileDescriptionAttribute.h"

#include "ImfPreviewImageAttribute.h"

#include "ImfChannelList.h"

#include "ImfMisc.h"

#include "ImfTiledMisc.h"

#include "ImfStdIO.h"

#include "ImfCompressor.h"

#include "ImfOutputStreamMutex.h"

#include "ImfOutputPartData.h"

#include "ImfArray.h"

#include "ImfXdr.h"

#include "ImfVersion.h"

#include "ImfTileOffsets.h"

#include "ImfThreading.h"

#include "ImfPartType.h"

#include "ImathBox.h"

#include "IlmThreadPool.h"

#include "IlmThreadSemaphore.h"

#include "IlmThreadMutex.h"

#include "Iex.h"

#include <string>

#include <vector>

#include <fstream>

#include <assert.h>

#include <map>

#include <algorithm>

#include "ImfNamespace.h"

OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER

using IMATH_NAMESPACE::Box2i;

using IMATH_NAMESPACE::V2i;

using std::string;

using std::vector;

using std::ofstream;

using std::map;

using std::min;

using std::max;

using std::swap;

using ILMTHREAD_NAMESPACE::Mutex;

using ILMTHREAD_NAMESPACE::Lock;

using ILMTHREAD_NAMESPACE::Semaphore;

using ILMTHREAD_NAMESPACE::Task;

using ILMTHREAD_NAMESPACE::TaskGroup;

using ILMTHREAD_NAMESPACE::ThreadPool;

namespace {

struct TOutSliceInfo

{

    PixelType                   type;

    const char *                base;

    size_t                      sampleStride;

    size_t                      xStride;

    size_t                      yStride;

    bool                        zero;

    int                         xTileCoords;

    int                         yTileCoords;

    TOutSliceInfo (PixelType type = HALF,

                   size_t sampleStride = 0,

                   size_t xStride = 0,

                   size_t yStride = 0,

                   bool zero = false,

                   int xTileCoords = 0,

                   int yTileCoords = 0);

};

TOutSliceInfo::TOutSliceInfo (PixelType t,

                              size_t spst,

                              size_t xStride,

                              size_t yStride,

                              bool z,

                              int xtc,

                              int ytc)

:

    type (t),

    sampleStride (spst),

    xStride(xStride),

    yStride(yStride),

    zero (z),

    xTileCoords (xtc),

    yTileCoords (ytc)

{

    // empty

}

struct TileCoord

{

    int         dx;

    int         dy;

    int         lx;

    int         ly;

    TileCoord (int xTile = 0, int yTile = 0,

               int xLevel = 0, int yLevel = 0)

    :

        dx (xTile),  dy (yTile),

        lx (xLevel), ly (yLevel)

    {

        // empty

    }

    bool

    operator < (const TileCoord &other) const

    {

        return (ly < other.ly) ||

               (ly == other.ly && lx < other.lx) ||

               ((ly == other.ly && lx == other.lx) &&

                    ((dy < other.dy) || (dy == other.dy && dx < other.dx)));

    }

    bool

    operator == (const TileCoord &other) const

    {

        return lx == other.lx &&

               ly == other.ly &&

               dx == other.dx &&

               dy == other.dy;

    }

};

struct BufferedTile

{

    char *      pixelData;

    Int64         pixelDataSize;

    Int64         unpackedDataSize;

    char *      sampleCountTableData;

    Int64         sampleCountTableSize;

    BufferedTile (const char *data, int size, int unpackedSize,

                  const char *tableData, int tableSize):

        pixelData (0),

        pixelDataSize(size),

        unpackedDataSize(unpackedSize),

        sampleCountTableData(0),

        sampleCountTableSize(tableSize)

    {

        pixelData = new char[pixelDataSize];

        memcpy (pixelData, data, pixelDataSize);

        sampleCountTableData = new char[tableSize];

        memcpy (sampleCountTableData, tableData, tableSize);

    }

    ~BufferedTile()

    {

        delete [] pixelData;

        delete [] sampleCountTableData;

    }

};

typedef map <TileCoord, BufferedTile *> TileMap;

struct TileBuffer

{

    Array<char>         buffer;

    const char *        dataPtr;

    Int64               dataSize;

    Int64               uncompressedSize;

    Compressor *        compressor;

    Array<char>         sampleCountTableBuffer;

    const char *        sampleCountTablePtr;

    Int64               sampleCountTableSize;

    Compressor*         sampleCountTableCompressor;

    TileCoord           tileCoord;

    bool                hasException;

    string              exception;

     TileBuffer ();

    ~TileBuffer ();

    inline void         wait () {_sem.wait();}

    inline void         post () {_sem.post();}

  protected:

    Semaphore           _sem;

};

TileBuffer::TileBuffer ():

    dataPtr (0),

    dataSize (0),

    compressor (0),

    sampleCountTablePtr (0),

    sampleCountTableCompressor (0),

    hasException (false),

    exception (),

    _sem (1)

{

    // empty

}

TileBuffer::~TileBuffer ()

{

    if (compressor != 0)

        delete compressor;

    if (sampleCountTableCompressor != 0)

        delete sampleCountTableCompressor;

}

} // namespace

struct DeepTiledOutputFile::Data

{

    Header              header;                 // the image header

    int                 version;                // file format version

    bool                multipart;              // file is multipart

    TileDescription     tileDesc;               // describes the tile layout

    DeepFrameBuffer     frameBuffer;            // framebuffer to write into

    Int64               previewPosition;

    LineOrder           lineOrder;              // the file's lineorder

    int                 minX;                   // data window's min x coord

    int                 maxX;                   // data window's max x coord

    int                 minY;                   // data window's min y coord

    int                 maxY;                   // data window's max x coord

    int                 numXLevels;             // number of x levels

    int                 numYLevels;             // number of y levels

    int *               numXTiles;              // number of x tiles at a level

    int *               numYTiles;              // number of y tiles at a level

    TileOffsets         tileOffsets;            // stores offsets in file for

                                                // each tile

    Compressor::Format  format;                 // compressor's data format

    vector<TOutSliceInfo*> slices;              // info about channels in file

    vector<TileBuffer*> tileBuffers;

    Int64               tileOffsetsPosition;    // position of the tile index

    TileMap             tileMap;                // the map of buffered tiles

    TileCoord           nextTileToWrite;

    int                 partNumber;             // the output part number

    char*               sampleCountSliceBase;   // the pointer to the number

                                                // of samples in each pixel

    int                 sampleCountXStride;     // the x stride for sampleCountSliceBase

    int                 sampleCountYStride;     // the y stride for sampleCountSliceBase

    int                 sampleCountXTileCoords; // using x coordinates relative to current tile

    int                 sampleCountYTileCoords; // using y coordinates relative to current tile

    Int64                 maxSampleCountTableSize;// the max size in bytes for a pixel

                                                // sample count table

    OutputStreamMutex*  _streamData;

    bool                _deleteStream;

     Data (int numThreads);

    ~Data ();

    inline TileBuffer * getTileBuffer (int number);

                                                // hash function from tile

                                                // buffer coords into our

                                                // vector of tile buffers

    int&                getSampleCount(int x, int y);

                                                // get the number of samples

                                                // in each pixel

    TileCoord           nextTileCoord (const TileCoord &a);

};

DeepTiledOutputFile::Data::Data (int numThreads):

    numXTiles(0),

    numYTiles(0),

    tileOffsetsPosition (0),

    partNumber(-1),

    _streamData(NULL),

    _deleteStream(true)

{

    //

    // We need at least one tileBuffer, but if threading is used,

    // to keep n threads busy we need 2*n tileBuffers

    //

    tileBuffers.resize (max (1, 2 * numThreads));

    for (size_t i = 0; i < tileBuffers.size(); i++)

        tileBuffers[i] = 0;

}

DeepTiledOutputFile::Data::~Data ()

{

    delete [] numXTiles;

    delete [] numYTiles;

    //

    // Delete all the tile buffers, if any still happen to exist

    //

    for (TileMap::iterator i = tileMap.begin(); i != tileMap.end(); ++i)

        delete i->second;

    for (size_t i = 0; i < tileBuffers.size(); i++)

        if (tileBuffers[i] != 0)

            delete tileBuffers[i];

    for (size_t i = 0; i < slices.size(); i++)

        delete slices[i];

}

int&

DeepTiledOutputFile::Data::getSampleCount(int x, int y)

{

    return sampleCount(sampleCountSliceBase,

                       sampleCountXStride,

                       sampleCountYStride,

                       x, y);

}

TileBuffer*

DeepTiledOutputFile::Data::getTileBuffer (int number)

{

    return tileBuffers[number % tileBuffers.size()];

}

TileCoord

DeepTiledOutputFile::Data::nextTileCoord (const TileCoord &a)

{

    TileCoord b = a;

    if (lineOrder == INCREASING_Y)

    {

        b.dx++;

        if (b.dx >= numXTiles[b.lx])

        {

            b.dx = 0;

            b.dy++;

            if (b.dy >= numYTiles[b.ly])

            {

                //

                // the next tile is in the next level

                //

                b.dy = 0;

                switch (tileDesc.mode)

                {

                  case ONE_LEVEL:

                  case MIPMAP_LEVELS:

                    b.lx++;

                    b.ly++;

                    break;

                  case RIPMAP_LEVELS:

                    b.lx++;

                    if (b.lx >= numXLevels)

                    {

                        b.lx = 0;

                        b.ly++;

                        #ifdef DEBUG

                            assert (b.ly <= numYLevels);

                        #endif

                    }

                    break;

                  case NUM_LEVELMODES :

                      throw IEX_NAMESPACE::LogicExc("unknown level mode computing nextTileCoord");

                }

            }

        }

    }

    else if (lineOrder == DECREASING_Y)

    {

        b.dx++;

        if (b.dx >= numXTiles[b.lx])

        {

            b.dx = 0;

            b.dy--;

            if (b.dy < 0)

            {

                //

                // the next tile is in the next level

                //

                switch (tileDesc.mode)

                {

                  case ONE_LEVEL:

                  case MIPMAP_LEVELS:

                    b.lx++;

                    b.ly++;

                    break;

                  case RIPMAP_LEVELS:

                    b.lx++;

                    if (b.lx >= numXLevels)

                    {

                        b.lx = 0;

                        b.ly++;

                        #ifdef DEBUG

                            assert (b.ly <= numYLevels);

                        #endif

                    }

                    break;

                  case NUM_LEVELMODES :

                      throw IEX_NAMESPACE::LogicExc("unknown level mode computing nextTileCoord");

                }

                if (b.ly < numYLevels)

                    b.dy = numYTiles[b.ly] - 1;

            }

        }

    }else if(lineOrder==RANDOM_Y)

    {                 

        THROW (IEX_NAMESPACE::ArgExc,

              "can't compute next tile from randomly ordered image: use getTilesInOrder instead");

    }

    return b;

}

namespace {

void

writeTileData (DeepTiledOutputFile::Data *ofd,

               int dx, int dy,

               int lx, int ly,

               const char pixelData[],

               Int64 pixelDataSize,

               Int64 unpackedDataSize,

               const char sampleCountTableData[],

               Int64 sampleCountTableSize)

{

    //

    // Store a block of pixel data in the output file, and try

    // to keep track of the current writing position the file,

    // without calling tellp() (tellp() can be fairly expensive).

    //

    Int64 currentPosition = ofd->_streamData->currentPosition;

    ofd->_streamData->currentPosition = 0;

    if (currentPosition == 0)

        currentPosition = ofd->_streamData->os->tellp();

    ofd->tileOffsets (dx, dy, lx, ly) = currentPosition;

    #ifdef DEBUG

        assert (ofd->_streamData->os->tellp() == currentPosition);

    #endif

    //

    // Write the tile header.

    //

    if (ofd->multipart)

    {

        Xdr::write <StreamIO> (*ofd->_streamData->os, ofd->partNumber);

    }

    Xdr::write <StreamIO> (*ofd->_streamData->os, dx);

    Xdr::write <StreamIO> (*ofd->_streamData->os, dy);

    Xdr::write <StreamIO> (*ofd->_streamData->os, lx);

    Xdr::write <StreamIO> (*ofd->_streamData->os, ly);

    //

    // Write the packed size of the pixel sample count table (64 bits)

    //

    Xdr::write <StreamIO> (*ofd->_streamData->os, sampleCountTableSize);

    //

    // Write the packed and unpacked data size (64 bits each)

    //

    Xdr::write <StreamIO> (*ofd->_streamData->os, pixelDataSize);

    Xdr::write <StreamIO> (*ofd->_streamData->os, unpackedDataSize);

    //

    // Write the compressed pixel sample count table.

    //

    ofd->_streamData->os->write (sampleCountTableData, sampleCountTableSize);

    //

    // Write the compressed data.

    //

    ofd->_streamData->os->write (pixelData, pixelDataSize);

    //

    // Keep current position in the file so that we can avoid

    // redundant seekg() operations (seekg() can be fairly expensive).

    //

    ofd->_streamData->currentPosition = currentPosition        +

                                  4 * Xdr::size<int>()   + // dx, dy, lx, ly,

                                  3 * Xdr::size<Int64>() + // sampleCountTableSize,

                                                           // pixelDataSize,

                                                           // unpackedDataSize

                                  sampleCountTableSize   +

                                  pixelDataSize;

    if (ofd->multipart)

    {

        ofd->_streamData->currentPosition += Xdr::size<int>();

    }

}

void

bufferedTileWrite (

                   DeepTiledOutputFile::Data *ofd,

                   int dx, int dy,

                   int lx, int ly,

                   const char pixelData[],

                   Int64 pixelDataSize,

                   Int64 unpackedDataSize,

                   const char sampleCountTableData[],

                   Int64 sampleCountTableSize)

{

    //

    // Check if a tile with coordinates (dx,dy,lx,ly) has already been written.

    //

    if (ofd->tileOffsets (dx, dy, lx, ly))

    {

        THROW (IEX_NAMESPACE::ArgExc,

               "Attempt to write tile "

               "(" << dx << ", " << dy << ", " << lx << ", " << ly << ") "

               "more than once.");

    }

    //

    // If tiles can be written in random order, then don't buffer anything.

    //

    if (ofd->lineOrder == RANDOM_Y)

    {

        writeTileData (ofd, dx, dy, lx, ly,

                       pixelData, pixelDataSize, unpackedDataSize,

                       sampleCountTableData, sampleCountTableSize);

        return;

    }

    //

    // If the tiles cannot be written in random order, then check if a

    // tile with coordinates (dx,dy,lx,ly) has already been buffered.

    //

    TileCoord currentTile = TileCoord(dx, dy, lx, ly);

    if (ofd->tileMap.find (currentTile) != ofd->tileMap.end())

    {

        THROW (IEX_NAMESPACE::ArgExc,

               "Attempt to write tile "

               "(" << dx << ", " << dy << ", " << lx << ", " << ly << ") "

               "more than once.");

    }

    //

    // If all the tiles before this one have already been written to the file,

    // then write this tile immediately and check if we have buffered tiles

    // that can be written after this tile.

    //

    // Otherwise, buffer the tile so it can be written to file later.

    //

    if (ofd->nextTileToWrite == currentTile)

    {

        writeTileData (ofd, dx, dy, lx, ly,

                       pixelData, pixelDataSize, unpackedDataSize,

                       sampleCountTableData, sampleCountTableSize);

        ofd->nextTileToWrite = ofd->nextTileCoord (ofd->nextTileToWrite);

        TileMap::iterator i = ofd->tileMap.find (ofd->nextTileToWrite);

        //

        // Step through the tiles and write all successive buffered tiles after

        // the current one.

        //

        while(i != ofd->tileMap.end())

        {

            //

            // Write the tile, and then delete the tile's buffered data

            //

            writeTileData (ofd,

                           i->first.dx, i->first.dy,

                           i->first.lx, i->first.ly,

                           i->second->pixelData,

                           i->second->pixelDataSize,

                           i->second->unpackedDataSize,

                           i->second->sampleCountTableData,

                           i->second->sampleCountTableSize);

            delete i->second;

            ofd->tileMap.erase (i);

            //

            // Proceed to the next tile

            //

            ofd->nextTileToWrite = ofd->nextTileCoord (ofd->nextTileToWrite);

            i = ofd->tileMap.find (ofd->nextTileToWrite);

        }

    }

    else

    {

        //

        // Create a new BufferedTile, copy the pixelData into it, and

        // insert it into the tileMap.

        //

        ofd->tileMap[currentTile] =

            new BufferedTile ((const char *)pixelData, pixelDataSize, unpackedDataSize,

                              sampleCountTableData, sampleCountTableSize);

    }

}

void

convertToXdr (DeepTiledOutputFile::Data *ofd,

              Array<char>& tileBuffer,

              int numScanLines,

              vector<Int64>& bytesPerLine)

{

    //

    // Convert the contents of a TiledOutputFile's tileBuffer from the

    // machine's native representation to Xdr format. This function is called

    // by writeTile(), below, if the compressor wanted its input pixel data

    // in the machine's native format, but then failed to compress the data

    // (most compressors will expand rather than compress random input data).

    //

    // Note that this routine assumes that the machine's native representation

    // of the pixel data has the same size as the Xdr representation.  This

    // makes it possible to convert the pixel data in place, without an

    // intermediate temporary buffer.

    //

    //

    // Set these to point to the start of the tile.

    // We will write to toPtr, and read from fromPtr.

    //

    char *writePtr = tileBuffer;

    const char *readPtr = writePtr;

    //

    // Iterate over all scan lines in the tile.

    //

    for (int y = 0; y < numScanLines; ++y)

    {

        //

        // Iterate over all slices in the file.

        //

        for (unsigned int i = 0; i < ofd->slices.size(); ++i)

        {

            const TOutSliceInfo &slice = *ofd->slices[i];

            //

            // Convert the samples in place.

            //

            Int64 numPixelsPerScanLine = bytesPerLine[y];

            convertInPlace (writePtr, readPtr, slice.type,

                            numPixelsPerScanLine);

        }

    }

    #ifdef DEBUG

        assert (writePtr == readPtr);

    #endif

}

//

// A TileBufferTask encapsulates the task of copying a tile from

// the user's framebuffer into a LineBuffer and compressing the data

// if necessary.

//

class TileBufferTask: public Task

{

  public:

    TileBufferTask (TaskGroup *group,

                    DeepTiledOutputFile::Data *ofd,

                    int number,

                    int dx, int dy,

                    int lx, int ly);

    virtual ~TileBufferTask ();

    virtual void                execute ();

  private:

    DeepTiledOutputFile::Data *     _ofd;

    TileBuffer *                _tileBuffer;

};

TileBufferTask::TileBufferTask

    (TaskGroup *group,

     DeepTiledOutputFile::Data *ofd,

     int number,

     int dx, int dy,

     int lx, int ly)

:

    Task (group),

    _ofd (ofd),

    _tileBuffer (_ofd->getTileBuffer (number))

{

    //

    // Wait for the tileBuffer to become available

    //

    _tileBuffer->wait ();

    _tileBuffer->tileCoord = TileCoord (dx, dy, lx, ly);

}

TileBufferTask::~TileBufferTask ()

{

    //

    // Signal that the tile buffer is now free

    //

    _tileBuffer->post ();

}

void

TileBufferTask::execute ()

{

    try

    {

        //

        // First copy the pixel data from the frame buffer

        // into the tile buffer

        //

        // Convert one tile's worth of pixel data to

        // a machine-independent representation, and store

        // the result in _tileBuffer->buffer.

        //

        Box2i tileRange = OPENEXR_IMF_INTERNAL_NAMESPACE::dataWindowForTile (

                _ofd->tileDesc,

                _ofd->minX, _ofd->maxX,

                _ofd->minY, _ofd->maxY,

                _tileBuffer->tileCoord.dx,

                _tileBuffer->tileCoord.dy,

                _tileBuffer->tileCoord.lx,

                _tileBuffer->tileCoord.ly);

        int numScanLines = tileRange.max.y - tileRange.min.y + 1;

//        int numPixelsPerScanLine = tileRange.max.x - tileRange.min.x + 1;

        //

        // Get the bytes for each line.

        //

        vector<Int64> bytesPerLine(_ofd->tileDesc.ySize);

        vector<int> xOffsets(_ofd->slices.size());

        vector<int> yOffsets(_ofd->slices.size());

        for (size_t i = 0; i < _ofd->slices.size(); i++)

        {

            const TOutSliceInfo &slice = *_ofd->slices[i];

            xOffsets[i] = slice.xTileCoords * tileRange.min.x;

            yOffsets[i] = slice.yTileCoords * tileRange.min.y;

        }

        calculateBytesPerLine(_ofd->header,

                              _ofd->sampleCountSliceBase,

                              _ofd->sampleCountXStride,

                              _ofd->sampleCountYStride,

                              tileRange.min.x, tileRange.max.x,

                              tileRange.min.y, tileRange.max.y,

                              xOffsets, yOffsets,

                              bytesPerLine);

        //

        // Allocate the memory for internal buffer.

        // (TODO) more efficient memory management?

        //

        Int64 totalBytes = 0;

        Int64 maxBytesPerTileLine = 0;

        for (size_t i = 0; i < bytesPerLine.size(); i++)

        {

            totalBytes += bytesPerLine[i];

            if (bytesPerLine[i] > maxBytesPerTileLine)

                maxBytesPerTileLine = bytesPerLine[i];

        }

        _tileBuffer->buffer.resizeErase(totalBytes);

        char *writePtr = _tileBuffer->buffer;

        //

        // Iterate over the scan lines in the tile.

        //