From 67c48b4dc1b27bf0b748c8ac2aa50d527b055b00 Mon Sep 17 00:00:00 2001 From: Packit Service Date: Dec 16 2020 10:08:31 +0000 Subject: Source-git repo for imports/c8/libaec-1.0.2-3.el8 --- diff --git a/.packit.yaml b/.packit.yaml index 6b5a93b..c59a8f8 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -1,12 +1,11 @@ jobs: - job: copr_build metadata: - targets: &id001 - - centos-stream-x86_64 + targets: &id001 [centos-stream-x86_64] trigger: pull_request - job: tests metadata: targets: *id001 trigger: pull_request specfile_path: SPECS/libaec.spec -upstream_ref: sg-start +upstream_ref: c8-source-git diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d1aa111 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,14 @@ +Mathis Rosenhauer, +Moritz Hanke, +Joerg Behrens + +Deutsches Klimarechenzentrum GmbH +Bundesstr. 45a +20146 Hamburg +Germany + +Luis Kornblueh +Max-Planck-Institut fuer Meteorologie +Bundesstr. 53 +20146 Hamburg +Germany diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..80b1b1e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,87 @@ +# libaec Changelog +All notable changes to libaec will be documented in this file. + +## [1.0.2] - 2017-10-18 + +### Fixed +- C99 requirement in all build systems + +## [1.0.1] - 2017-07-14 + +### Fixed +- Potential security vulnerabilities in decoder exposed by libFuzzer. + +### Added +- Fuzz target for decoding and encoding. + +### Changed +- Improved Cmake support by Christoph Junghans + +## [1.0.0] - 2016-11-16 + +### Added +- Include CCSDS test data with libaec. See THANKS. + +### Changed +- Better compatibility with OSX for make check. +- Allow Cygwin to build DLLs. + +## [0.3.4] - 2016-08-16 + +### Fixed +- Pad incomplete last line when in SZ compatibility mode. + +## [0.3.3] - 2016-05-12 + +### Fixed +- Bug with zero blocks in the last RSI (reference sample interval) +when data size is not a multiple of RSIs or segments (64 blocks) and +the zero region reaches a segment boundary. +- More robust error handling. + +### Changed +- Vectorization improvement for Intel compiler. +- Better compatibility with netcdf's build process. + +## [0.3.2] - 2015-02-04 + +### Changed +- Allow nonconforming block sizes in SZ mode. +- Performance improvement for decoder. + +## [0.3.1] - 2014-10-23 + +### Fixed +- Allow incomplete scanlines in SZ mode. + +## [0.3] - 2014-08-06 + +### Changed +- Performance improvement for encoding pre-precessed data. +- More efficient coding of second extension if reference sample is +present. +- Port library to Windows (Visual Studio). + +### Added +- Support building with CMake. +- Benchmarking target using ECHAM data (make bench). + +## [0.2] - 2014-02-12 + +### Fixed +- Incorrect length calculation in assessment of Second Extension +coding. +- Unlimited encoding of fundamental sequences. +- Handle corrupted compressed data more gracefully. + +### Added +- Additional testing with official CCSDS sample data. +- Support restricted coding options from latest standard. + +### Changed +- Facilitate generation of SIMD instructions by compiler. + +## [0.1] - 2013-05-21 + +### Added +- Initial release. diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..3303f72 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,86 @@ +cmake_minimum_required(VERSION 3.1) +set(CMAKE_C_STANDARD 99) +include(CheckIncludeFiles) +include(TestBigEndian) +include(CheckCSourceCompiles) +include(CheckSymbolExists) +include(cmake/macros.cmake) +project(libaec) +set(libaec_VERSION_MAJOR 1) +set(libaec_VERSION_MINOR 0) +set(libaec_VERSION_PATCH 2) +set(CMAKE_BUILD_TYPE Release) +enable_testing() + +check_include_files(malloc.h HAVE_MALLOC_H) +check_include_files(stdint.h HAVE_STDINT_H) +test_big_endian(WORDS_BIGENDIAN) +check_clzll(HAVE_DECL___BUILTIN_CLZLL) +if(NOT HAVE_DECL___BUILTIN_CLZLL) + check_bsr64(HAVE_BSR64) +endif(NOT HAVE_DECL___BUILTIN_CLZLL) +find_inline_keyword() +find_restrict_keyword() + +check_symbol_exists(snprintf "stdio.h" HAVE_SNPRINTF) +if(NOT HAVE_SNPRINTF) + check_symbol_exists(_snprintf "stdio.h" HAVE__SNPRINTF) + check_symbol_exists(_snprintf_s "stdio.h" HAVE__SNPRINTF_S) +endif(NOT HAVE_SNPRINTF) + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.h.in + ${CMAKE_CURRENT_BINARY_DIR}/config.h) +add_definitions("-DHAVE_CONFIG_H") + +option(BUILD_SHARED_LIBS "Build Shared Libraries" ON) +if(BUILD_SHARED_LIBS) + set (LIB_TYPE SHARED) +else(BUILD_SHARED_LIBS) + if(WIN32) + message(WARNING "You will have to modify libaec.h for static libs.") + endif(WIN32) + set(LIB_TYPE STATIC) +endif(BUILD_SHARED_LIBS) + +set(libaec_SRCS + ${PROJECT_SOURCE_DIR}/src/encode.c + ${PROJECT_SOURCE_DIR}/src/encode_accessors.c + ${PROJECT_SOURCE_DIR}/src/decode.c) + +include_directories("${PROJECT_BINARY_DIR}") +include_directories("${PROJECT_SOURCE_DIR}/src") + +add_subdirectory(src) +add_subdirectory(tests) + +option(AEC_FUZZING "Enable build of fuzzing targets" OFF) +if(AEC_FUZZING) + add_subdirectory(fuzzing) +endif() + +set(CPACK_PACKAGE_NAME "libaec") +set(CPACK_PACKAGE_VENDOR "Deutsches Klimarechenzentrum GmbH") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY + "libaec - Adaptive Entropy Coding library") +set(CPACK_PACKAGE_VERSION_MAJOR "${libaec_VERSION_MAJOR}") +set(CPACK_PACKAGE_VERSION_MINOR "${libaec_VERSION_MINOR}") +set(CPACK_PACKAGE_VERSION_PATCH "${libaec_VERSION_PATCH}") +set(CPACK_PACKAGE_INSTALL_DIRECTORY "libaec") +set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/Copyright.txt") + +if(WIN32) + set(CPACK_GENERATOR "WIX") + set(CPACK_WIX_UPGRADE_GUID "E58A21F6-BB99-46B5-8AD8-6114E4CCA0A6") + if(CMAKE_CL_64) + set(CPACK_INSTALL_DIRECTORY "$PROGRAMFILES64") + set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY + "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION} (Win64)") + else(CMAKE_CL_64) + set(CPACK_INSTALL_DIRECTORY "$PROGRAMFILES") + set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY + "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}") + endif(CMAKE_CL_64) +endif(WIN32) + +include(CPack) diff --git a/Copyright.txt b/Copyright.txt new file mode 100644 index 0000000..72969b0 --- /dev/null +++ b/Copyright.txt @@ -0,0 +1,37 @@ +Copyright 2012 - 2017 + +Mathis Rosenhauer, Moritz Hanke, Joerg Behrens +Deutsches Klimarechenzentrum GmbH +Bundesstr. 45a +20146 Hamburg +Germany + +Luis Kornblueh +Max-Planck-Institut fuer Meteorologie +Bundesstr. 53 +20146 Hamburg +Germany + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..85c67d5 --- /dev/null +++ b/INSTALL @@ -0,0 +1,95 @@ +Getting libaec +============== + +The source code of libaec is hosted at DKRZ GitLab. + +## Source code and binary releases + +The latest releases of libaec can be downloaded at the following +location: + + https://gitlab.dkrz.de/k202009/libaec/tags + +## Developer snapshot + + git clone https://gitlab.dkrz.de/k202009/libaec.git + + +Installation +============ + +## General considerations + +Libaec achieves the best performance on 64 bit systems. The library +will work correctly on 32 bit systems but encoding and decoding +performance will be much lower. + +## Installation from source code release with configure + +The most common installation procedure on Unix-like systems looks as +follows: + +Unpack the tar archive and change into the unpacked directory. + + mkdir build + cd build + ../configure + make check install + +## Installation from source code release with CMake + +As an alternative, you can use CMake to install libaec. + +Unpack the tar archive and change into the unpacked directory. + + mkdir build + cd build + cmake .. + make install + +You can set options for compiling using the CMake GUI by replacing the cmake +command with + + cmake-gui .. + +or by setting the options manually, e.g. + + cmake -DCMAKE_INSTALL_PREFIX=~/local .. + +in order to set the install prefix to ~/local + +CMake can also generate project files for Microsoft Visual Studio when +used in Windows. + +## Installation from cloned repository + +The configure script is not included in the repository. You can +generate it with autotools: + + cd libaec + autoreconv -iv + mkdir build + cd build + ../configure + +Also not included are CCSDS sample data which are needed for +testing. They have to be downloaded prior to running 'make check': + + make update-sampledata + make check install + + +Intel compiler settings +======================= + +The Intel compiler can improve performance by vectorizing certain +parts of the code on x86 architectures. Assuming your CPU supports +AVX2, the following options will increase encoding speed. + + ../configure CC=icc + make CFLAGS="-O3 -xCORE-AVX2" bench + +On a 3.4 GHz E3-1240 v3 we see more than 400 MiB/s for encoding +typical data. + +Using other SIMD instruction sets on older CPUs may also help. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..c262e41 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,15 @@ +ACLOCAL_AMFLAGS = -I m4 +SUBDIRS = src tests +EXTRA_DIST = doc/patent.txt CMakeLists.txt cmake/config.h.in \ +cmake/macros.cmake README.md README.SZIP CHANGELOG.md Copyright.txt data + +sampledata = 121B2TestData +sampledata_url = https://cwe.ccsds.org/sls/docs/SLS-DC/BB121B2TestData/$(sampledata).zip +update-sampledata: + cd $(srcdir)/data && \ + rm -rf $(sampledata) && \ + curl $(sampledata_url) -O && unzip $(sampledata).zip && \ + rm -f $(sampledata).zip && \ + cd $(sampledata)/AllOptions && mv -f test_P512n22.dat test_p512n22.dat + +.PHONY: update-sampledata diff --git a/README.SZIP b/README.SZIP new file mode 100644 index 0000000..c62b3b6 --- /dev/null +++ b/README.SZIP @@ -0,0 +1,15 @@ +********************************************************************** + SZIP compatibility +********************************************************************** + +Libaec includes a free drop-in replacement for the SZIP +library[1]. Just replace SZIP's shared library libsz.so* with +libaec.so* and libsz.so* from libaec. For Windows the DLLs are called +SZIP.DLL and AEC.DLL. Code which is dynamically linked with SZIP such +as HDF5 should continue to work with libaec. No re-compilation +required. + +HDF5 files which contain SZIP encoded data can be decoded by HDF5 +using libaec and vice versa. + +[1] http://www.hdfgroup.org/doc_resource/SZIP/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b0817e --- /dev/null +++ b/README.md @@ -0,0 +1,225 @@ +# libaec - Adaptive Entropy Coding library + +Libaec provides fast lossless compression of 1 up to 32 bit wide +signed or unsigned integers (samples). The library achieves best +results for low entropy data as often encountered in space imaging +instrument data or numerical model output from weather or climate +simulations. While floating point representations are not directly +supported, they can also be efficiently coded by grouping exponents +and mantissa. + +Libaec implements +[Golomb-Rice](http://en.wikipedia.org/wiki/Golomb_coding) coding as +defined in the Space Data System Standard documents [121.0-B-2][1] and +[120.0-G-2][2]. + +## Downloads + +Source code and binary installer can be [downloaded here](https://gitlab.dkrz.de/k202009/libaec/tags). + +## Patent + +In [patent.txt](doc/patent.txt) a statement on potentially +applying intellectual property rights is given. + +## Installation + +See [INSTALL](INSTALL) for details. + +## SZIP Compatibility + +[Libaec can replace SZIP](README.SZIP). + +## Encoding + +In this context efficiency refers to the size of the encoded +data. Performance refers to the time it takes to encode data. + +Suppose you have an array of 32 bit signed integers you want to +compress. The pointer pointing to the data shall be called `*source`, +output goes into `*dest`. + +```c +#include + +... + struct aec_stream strm; + int32_t *source; + unsigned char *dest; + + /* input data is 32 bits wide */ + strm.bits_per_sample = 32; + + /* define a block size of 16 */ + strm.block_size = 16; + + /* the reference sample interval is set to 128 blocks */ + strm.rsi = 128; + + /* input data is signed and needs to be preprocessed */ + strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS; + + /* pointer to input */ + strm.next_in = (unsigned char *)source; + + /* length of input in bytes */ + strm.avail_in = source_length * sizeof(int32_t); + + /* pointer to output buffer */ + strm.next_out = dest; + + /* length of output buffer in bytes */ + strm.avail_out = dest_length; + + /* initialize encoding */ + if (aec_encode_init(&strm) != AEC_OK) + return 1; + + /* Perform encoding in one call and flush output. */ + /* In this example you must be sure that the output */ + /* buffer is large enough for all compressed output */ + if (aec_encode(&strm, AEC_FLUSH) != AEC_OK) + return 1; + + /* free all resources used by encoder */ + aec_encode_end(&strm); +... +``` + +`block_size` can vary from 8 to 64 samples. Smaller blocks allow the +compression to adapt more rapidly to changing source +statistics. Larger blocks create less overhead but can be less +efficient if source statistics change across the block. + +`rsi` sets the reference sample interval. A large RSI will improve +performance and efficiency. It will also increase memory requirements +since internal buffering is based on RSI size. A smaller RSI may be +desirable in situations where each RSI will be packetized and possible +error propagation has to be minimized. + +### Flags: + +* `AEC_DATA_SIGNED`: input data are signed integers. Specifying this + correctly increases compression efficiency. Default is unsigned. + +* `AEC_DATA_PREPROCESS`: preprocessing input will improve compression + efficiency if data samples are correlated. It will only cost + performance for no gain in efficiency if the data is already + uncorrelated. + +* `AEC_DATA_MSB`: input data is stored most significant byte first + i.e. big endian. You have to specify `AEC_DATA_MSB` even if your host + architecture is big endian. Default is little endian on all + architectures. + +* `AEC_DATA_3BYTE`: the 17 to 24 bit input data is stored in three + bytes. This flag has no effect for other sample sizes. + +* `AEC_RESTRICTED`: use a restricted set of code options. This option is + only valid for `bits_per_sample` <= 4. + +* `AEC_PAD_RSI`: assume that the encoded RSI is padded to the next byte + boundary while decoding. The preprocessor macro `ENABLE_RSI_PADDING` + needs to be defined while compiling for the encoder to honour this + flag. + +### Data size: + +The following rules apply for deducing storage size from sample size +(`bits_per_sample`): + + **sample size** | **storage size** +--- | --- + 1 - 8 bits | 1 byte + 9 - 16 bits | 2 bytes +17 - 24 bits | 3 bytes (only if `AEC_DATA_3BYTE` is set) +25 - 32 bits | 4 bytes (if `AEC_DATA_3BYTE` is set) +17 - 32 bits | 4 bytes (if `AEC_DATA_3BYTE` is not set) + +If a sample requires less bits than the storage size provides, then +you have to make sure that unused bits are not set. Libaec does not +check this for performance reasons and will produce undefined output +if unused bits are set. All input data must be a multiple of the +storage size in bytes. Remaining bytes which do not form a complete +sample will be ignored. + +Libaec accesses `next_in` and `next_out` buffers only bytewise. There +are no alignment requirements for these buffers. + +### Flushing: + +`aec_encode` can be used in a streaming fashion by chunking input and +output and specifying `AEC_NO_FLUSH`. The function will return if either +the input runs empty or the output buffer is full. The calling +function can check `avail_in` and `avail_out` to see what occurred. The +last call to `aec_encode()` must set `AEC_FLUSH` to drain all +output. [aec.c](src/aec.c) is an example of streaming usage of encoding and +decoding. + +### Output: + +Encoded data will be written to the buffer submitted with +`next_out`. The length of the compressed data is `total_out`. + +See libaec.h for a detailed description of all relevant structure +members and constants. + + +## Decoding + +Using decoding is very similar to encoding, only the meaning of input +and output is reversed. + +```c +#include + +... + struct aec_stream strm; + /* this is now the compressed data */ + unsigned char *source; + /* here goes the uncompressed result */ + int32_t *dest; + + strm.bits_per_sample = 32; + strm.block_size = 16; + strm.rsi = 128; + strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS; + strm.next_in = source; + strm.avail_in = source_length; + strm.next_out = (unsigned char *)dest; + strm.avail_out = dest_lenth * sizeof(int32_t); + if (aec_decode_init(&strm) != AEC_OK) + return 1; + if (aec_decode(&strm, AEC_FLUSH) != AEC_OK) + return 1; + aec_decode_end(&strm); +... +``` + +It is strongly recommended that the size of the output buffer +(`next_out`) is a multiple of the storage size in bytes. If the buffer +is not a multiple of the storage size and the buffer gets filled to +the last sample, the error code `AEC_MEM_ERROR` is returned. + +It is essential for decoding that parameters like `bits_per_sample`, +`block_size`, `rsi`, and `flags` are exactly the same as they were for +encoding. Libaec does not store these parameters in the coded stream +so it is up to the calling program to keep the correct parameters +between encoding and decoding. + +The actual values of coding parameters are in fact only relevant for +efficiency and performance. Data integrity only depends on consistency +of the parameters. + + +## References + +[Consultative Committee for Space Data Systems. Lossless Data +Compression. Recommendation for Space Data System Standards, CCSDS +121.0-B-2. Blue Book. Issue 2. Washington, D.C.: CCSDS, May 2012.][1] +[1]: http://public.ccsds.org/publications/archive/121x0b2.pdf + +[Consultative Committee for Space Data Systems. Lossless Data +Compression. Recommendation for Space Data System Standards, CCSDS +120.0-G-3. Green Book. Issue 3. Washington, D.C.: CCSDS, April 2013.][2] +[2]: http://public.ccsds.org/publications/archive/120x0g3.pdf diff --git a/THANKS b/THANKS new file mode 100644 index 0000000..8281b0c --- /dev/null +++ b/THANKS @@ -0,0 +1,16 @@ +********************************************************************** + CCSDS sample data for libaec testing +********************************************************************** + +Thanks to the chairman of CCSDS SLS-MHDC, Aaron Kiely, to let us +distribute BB121B2 test data with libaec. The data files are used +during 'make check'. + +The most recent version of the data is available from the CCSDS web +site: + +http://cwe.ccsds.org/sls/docs/Forms/AllItems.aspx?RootFolder=%2Fsls%2Fdocs%2FSLS%2DDC%2FBB121B2TestData& + +You can also update the data included in the distribution with + + make update-sampledata diff --git a/cmake/config.h.in b/cmake/config.h.in new file mode 100644 index 0000000..d5783e0 --- /dev/null +++ b/cmake/config.h.in @@ -0,0 +1,8 @@ +#cmakedefine HAVE_MALLOC_H 1 +#cmakedefine HAVE_STDINT_H 1 +#cmakedefine WORDS_BIGENDIAN 1 +#cmakedefine HAVE_DECL___BUILTIN_CLZLL 1 +#cmakedefine HAVE_BSR64 1 +#cmakedefine HAVE_SNPRINTF 1 +#cmakedefine HAVE__SNPRINTF 1 +#cmakedefine HAVE__SNPRINTF_S 1 diff --git a/cmake/macros.cmake b/cmake/macros.cmake new file mode 100644 index 0000000..a568b15 --- /dev/null +++ b/cmake/macros.cmake @@ -0,0 +1,77 @@ +macro(check_clzll VARIABLE) + check_c_source_compiles( + "int main(int argc, char *argv[]) +{return __builtin_clzll(1LL);}" + ${VARIABLE} + ) +endmacro() + +macro(check_bsr64 VARIABLE) + check_c_source_compiles( + "int main(int argc, char *argv[]) +{unsigned long foo; unsigned __int64 bar=1LL; +return _BitScanReverse64(&foo, bar);}" + ${VARIABLE} + ) +endmacro() + +macro(find_inline_keyword) + #Inspired from http://www.cmake.org/Wiki/CMakeTestInline + set(INLINE_TEST_SRC "/* Inspired by autoconf's c.m4 */ +static inline int static_foo(){return 0\;} +int main(int argc, char *argv[]){return 0\;} +") + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCInline.c + ${INLINE_TEST_SRC}) + + foreach(KEYWORD "inline" "__inline__" "__inline") + if(NOT DEFINED C_INLINE) + try_compile(C_HAS_${KEYWORD} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCInline.c + COMPILE_DEFINITIONS "-Dinline=${KEYWORD}" + ) + if(C_HAS_${KEYWORD}) + set(C_INLINE TRUE) + add_definitions("-Dinline=${KEYWORD}") + message(STATUS "Inline keyword found - ${KEYWORD}") + endif(C_HAS_${KEYWORD}) + endif(NOT DEFINED C_INLINE) + endforeach(KEYWORD) + + if(NOT DEFINED C_INLINE) + add_definitions("-Dinline=") + message(STATUS "Inline keyword - not found") + endif(NOT DEFINED C_INLINE) +endmacro(find_inline_keyword) + +macro(find_restrict_keyword) + set(RESTRICT_TEST_SRC "/* Inspired by autoconf's c.m4 */ +int foo (int * restrict ip){return ip[0]\;} +int main(int argc, char *argv[]){int s[1]\; +int * restrict t = s\; t[0] = 0\; return foo(t)\;} +") + + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCRestrict.c + ${RESTRICT_TEST_SRC}) + + foreach(KEYWORD "restrict" "__restrict" "__restrict__" "_Restrict") + if(NOT DEFINED C_RESTRICT) + try_compile(C_HAS_${KEYWORD} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCRestrict.c + COMPILE_DEFINITIONS "-Drestrict=${KEYWORD}" + ) + if(C_HAS_${KEYWORD}) + set(C_RESTRICT TRUE) + add_definitions("-Drestrict=${KEYWORD}") + message(STATUS "Restrict keyword found - ${KEYWORD}") + endif(C_HAS_${KEYWORD}) + endif(NOT DEFINED C_RESTRICT) + endforeach(KEYWORD) + + if(NOT DEFINED C_RESTRICT) + add_definitions("-Drestrict=") + message(STATUS "Restrict keyword - not found") + endif(NOT DEFINED C_RESTRICT) +endmacro(find_restrict_keyword) diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..e958719 --- /dev/null +++ b/configure.ac @@ -0,0 +1,34 @@ +AC_PREREQ([2.64]) +AC_INIT([libaec], [1.0.2], [rosenhauer@dkrz.de]) + +AC_CONFIG_MACRO_DIR([m4]) +AC_CONFIG_AUX_DIR([config]) +AC_CONFIG_SRCDIR([src/libaec.h]) +AC_CONFIG_HEADERS(config/config.h) + +LT_INIT +gl_VISIBILITY +AM_INIT_AUTOMAKE([foreign]) +AM_MAINTAINER_MODE([enable]) + +AC_PROG_CC_C99 + +AC_HEADER_STDC + +AC_C_BIGENDIAN +AC_C_INLINE +AC_TYPE_INT64_T +AC_TYPE_SIZE_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_UINT8_T +AC_C_INLINE +AC_C_RESTRICT + +AC_CHECK_FUNCS([memset strstr snprintf]) +AC_CHECK_DECLS(__builtin_clzll) + +AM_EXTRA_RECURSIVE_TARGETS([bench benc bdec]) + +AC_CONFIG_FILES([Makefile src/Makefile tests/Makefile]) +AC_OUTPUT diff --git a/data/typical.rz b/data/typical.rz new file mode 100644 index 0000000..fc1e827 Binary files /dev/null and b/data/typical.rz differ diff --git a/doc/README b/doc/README new file mode 100644 index 0000000..4a96d8e --- /dev/null +++ b/doc/README @@ -0,0 +1,13 @@ +--------------------- + 32 bits vs. 64 bits +--------------------- + +libaec compiles and runs both in 32 and 64 bits. For encoding and +decoding I see a performance degradation with 32 bit binaries of 38% +and 33% resp. + +The main reason for this is that encoding and decoding use a buffer of +type uin64_t to align a potentially 32 bit wide sample with the output +(encoding) or input (decoding) stream. Avoiding the use of this +uin64_t buffer variable would degrade 64 bit performance for a modest +gain in 32 bit performance so I decided against it. diff --git a/doc/logo.svg b/doc/logo.svg new file mode 100644 index 0000000..ebe4e12 --- /dev/null +++ b/doc/logo.svg @@ -0,0 +1,114 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + C + AE + + Golomb Rice Adaptive Entropy Coder + + + diff --git a/doc/patent.txt b/doc/patent.txt new file mode 100644 index 0000000..c49e4bb --- /dev/null +++ b/doc/patent.txt @@ -0,0 +1,57 @@ +------------------------------------------------------------------- +Lossless Data Compression, Recommended Standard, Issue 2, May 2012, +CCSDS 121.0-B-2 +------------------------------------------------------------------- +Patent Considerations, Section A3, page 37 + +With respect to this issue we got the following information from +NASA's Chief Patent Counsel: + +From: Geurts, Bryan A. (GSFC-1401) +Sent: Tuesday, April 21, 2009 12:14 PM +To: Yeh, Pen-Shu (GSFC-5670) +Subject: RE: Need help from you again + +Pen-Shu, + +The license agreement with ICs, LLC, for Patent Numbers US 5,448,642, +US 5,687,255 and US 5,822,457, has been terminated effective March 31, +2009. NASA has no other licensee of these patents. It is NASA's intent +to not pay the remaining maintenance fees owing on these patents, +thereby allowing them to eventually become abandoned, and to not +enforce the patents. + +If you have any questions or concerns, please do not hesitate to contact me. + +Bryan A. Geurts +Chief Patent Counsel +NASA Goddard Space Flight Center +Code 140.1, 8800 Greenbelt Road +Greenbelt, MD 20771 +Phone: (301) 286-7352 +Fax: (301) 286-9502 + +Reassuring this information in August 2012 because of the patent +considerations in the above referenced CCSDS standard: + +From: Geurts, Bryan A. (GSFC-1401) +Sent: Friday, August 31, 2012 10:36 AM +To: Luis Kornblueh +Subject: RE: Patent 5448642 + +Dear Mr. Kornblueh + +Per my email cited above, NASA has no interest in licensing or +maintaining US Patent No. 5448642 and therefore has no objection to +your use of its teachings and claims. However, while NASA has +effectively abandoned this patent, any use of the subject matter is at +your own risk and NASA cannot warrant that its use will satisfy your +particular needs or that your use will not infringe on another's +intellectual property rights. + +------------------------------------------------------------------- + +The cited CCSDS standard does not refer to any other intellectual +property rights. + +------------------------------------------------------------------- diff --git a/fuzzing/CMakeLists.txt b/fuzzing/CMakeLists.txt new file mode 100644 index 0000000..b57e642 --- /dev/null +++ b/fuzzing/CMakeLists.txt @@ -0,0 +1,17 @@ +if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "You need to build with Clang for fuzzing to work") +endif() + +add_library(aec_fuzz STATIC ${libaec_SRCS}) +add_executable(fuzz_target fuzz_target.cc) + +set(FUZZ_TARGET_LINK_FLAGS "-fsanitize=address") +set(FUZZ_TARGET_COMPILE_FLAGS + "-g" + "-fsanitize=address" + "-fsanitize-coverage=trace-pc-guard,indirect-calls,edge") +set_target_properties(fuzz_target aec_fuzz PROPERTIES + LINK_FLAGS ${FUZZ_TARGET_LINK_FLAGS} + COMPILE_OPTIONS "${FUZZ_TARGET_COMPILE_FLAGS}") + +target_link_libraries(fuzz_target aec_fuzz Fuzzer) diff --git a/fuzzing/fuzz_target.cc b/fuzzing/fuzz_target.cc new file mode 100644 index 0000000..63fcf79 --- /dev/null +++ b/fuzzing/fuzz_target.cc @@ -0,0 +1,36 @@ +#include +#include +#include "libaec.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + struct aec_stream strm; + int32_t *source; + unsigned char *dest; + + if (Size < 2) + return 0; + + dest = (unsigned char *)malloc(Size * 4); + strm.bits_per_sample = (Data[0] & 0x1f) | 1; + strm.block_size = 8 << (Data[1] & 3); + strm.rsi = 2; + strm.flags = AEC_DATA_PREPROCESS; + if (Data[1] & 0x80) + strm.flags |= AEC_DATA_MSB; + if (Data[1] & 0x40) + strm.flags |= AEC_DATA_SIGNED; + if (strm.bits_per_sample <= 24 && + strm.bits_per_sample > 16 && + Data[1] & 0x10) + strm.flags |= AEC_DATA_3BYTE; + strm.next_in = (unsigned char *)(Data + 2); + strm.avail_in = Size - 2; + strm.next_out = dest; + strm.avail_out = (Size - 2) * 4; + if (Data[1] & 0x20) + aec_buffer_encode(&strm); + else + aec_buffer_decode(&strm); + free(dest); + return 0; +} diff --git a/include/libaec.h b/include/libaec.h new file mode 120000 index 0000000..ccf5359 --- /dev/null +++ b/include/libaec.h @@ -0,0 +1 @@ +../src/libaec.h \ No newline at end of file diff --git a/include/szlib.h b/include/szlib.h new file mode 120000 index 0000000..f74bf00 --- /dev/null +++ b/include/szlib.h @@ -0,0 +1 @@ +../src/szlib.h \ No newline at end of file diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/AUTHORS b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/AUTHORS deleted file mode 100644 index d1aa111..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/AUTHORS +++ /dev/null @@ -1,14 +0,0 @@ -Mathis Rosenhauer, -Moritz Hanke, -Joerg Behrens - -Deutsches Klimarechenzentrum GmbH -Bundesstr. 45a -20146 Hamburg -Germany - -Luis Kornblueh -Max-Planck-Institut fuer Meteorologie -Bundesstr. 53 -20146 Hamburg -Germany diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CHANGELOG.md b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CHANGELOG.md deleted file mode 100644 index 80b1b1e..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CHANGELOG.md +++ /dev/null @@ -1,87 +0,0 @@ -# libaec Changelog -All notable changes to libaec will be documented in this file. - -## [1.0.2] - 2017-10-18 - -### Fixed -- C99 requirement in all build systems - -## [1.0.1] - 2017-07-14 - -### Fixed -- Potential security vulnerabilities in decoder exposed by libFuzzer. - -### Added -- Fuzz target for decoding and encoding. - -### Changed -- Improved Cmake support by Christoph Junghans - -## [1.0.0] - 2016-11-16 - -### Added -- Include CCSDS test data with libaec. See THANKS. - -### Changed -- Better compatibility with OSX for make check. -- Allow Cygwin to build DLLs. - -## [0.3.4] - 2016-08-16 - -### Fixed -- Pad incomplete last line when in SZ compatibility mode. - -## [0.3.3] - 2016-05-12 - -### Fixed -- Bug with zero blocks in the last RSI (reference sample interval) -when data size is not a multiple of RSIs or segments (64 blocks) and -the zero region reaches a segment boundary. -- More robust error handling. - -### Changed -- Vectorization improvement for Intel compiler. -- Better compatibility with netcdf's build process. - -## [0.3.2] - 2015-02-04 - -### Changed -- Allow nonconforming block sizes in SZ mode. -- Performance improvement for decoder. - -## [0.3.1] - 2014-10-23 - -### Fixed -- Allow incomplete scanlines in SZ mode. - -## [0.3] - 2014-08-06 - -### Changed -- Performance improvement for encoding pre-precessed data. -- More efficient coding of second extension if reference sample is -present. -- Port library to Windows (Visual Studio). - -### Added -- Support building with CMake. -- Benchmarking target using ECHAM data (make bench). - -## [0.2] - 2014-02-12 - -### Fixed -- Incorrect length calculation in assessment of Second Extension -coding. -- Unlimited encoding of fundamental sequences. -- Handle corrupted compressed data more gracefully. - -### Added -- Additional testing with official CCSDS sample data. -- Support restricted coding options from latest standard. - -### Changed -- Facilitate generation of SIMD instructions by compiler. - -## [0.1] - 2013-05-21 - -### Added -- Initial release. diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CMakeLists.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CMakeLists.txt deleted file mode 100644 index 3303f72..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/CMakeLists.txt +++ /dev/null @@ -1,86 +0,0 @@ -cmake_minimum_required(VERSION 3.1) -set(CMAKE_C_STANDARD 99) -include(CheckIncludeFiles) -include(TestBigEndian) -include(CheckCSourceCompiles) -include(CheckSymbolExists) -include(cmake/macros.cmake) -project(libaec) -set(libaec_VERSION_MAJOR 1) -set(libaec_VERSION_MINOR 0) -set(libaec_VERSION_PATCH 2) -set(CMAKE_BUILD_TYPE Release) -enable_testing() - -check_include_files(malloc.h HAVE_MALLOC_H) -check_include_files(stdint.h HAVE_STDINT_H) -test_big_endian(WORDS_BIGENDIAN) -check_clzll(HAVE_DECL___BUILTIN_CLZLL) -if(NOT HAVE_DECL___BUILTIN_CLZLL) - check_bsr64(HAVE_BSR64) -endif(NOT HAVE_DECL___BUILTIN_CLZLL) -find_inline_keyword() -find_restrict_keyword() - -check_symbol_exists(snprintf "stdio.h" HAVE_SNPRINTF) -if(NOT HAVE_SNPRINTF) - check_symbol_exists(_snprintf "stdio.h" HAVE__SNPRINTF) - check_symbol_exists(_snprintf_s "stdio.h" HAVE__SNPRINTF_S) -endif(NOT HAVE_SNPRINTF) - -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.h.in - ${CMAKE_CURRENT_BINARY_DIR}/config.h) -add_definitions("-DHAVE_CONFIG_H") - -option(BUILD_SHARED_LIBS "Build Shared Libraries" ON) -if(BUILD_SHARED_LIBS) - set (LIB_TYPE SHARED) -else(BUILD_SHARED_LIBS) - if(WIN32) - message(WARNING "You will have to modify libaec.h for static libs.") - endif(WIN32) - set(LIB_TYPE STATIC) -endif(BUILD_SHARED_LIBS) - -set(libaec_SRCS - ${PROJECT_SOURCE_DIR}/src/encode.c - ${PROJECT_SOURCE_DIR}/src/encode_accessors.c - ${PROJECT_SOURCE_DIR}/src/decode.c) - -include_directories("${PROJECT_BINARY_DIR}") -include_directories("${PROJECT_SOURCE_DIR}/src") - -add_subdirectory(src) -add_subdirectory(tests) - -option(AEC_FUZZING "Enable build of fuzzing targets" OFF) -if(AEC_FUZZING) - add_subdirectory(fuzzing) -endif() - -set(CPACK_PACKAGE_NAME "libaec") -set(CPACK_PACKAGE_VENDOR "Deutsches Klimarechenzentrum GmbH") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY - "libaec - Adaptive Entropy Coding library") -set(CPACK_PACKAGE_VERSION_MAJOR "${libaec_VERSION_MAJOR}") -set(CPACK_PACKAGE_VERSION_MINOR "${libaec_VERSION_MINOR}") -set(CPACK_PACKAGE_VERSION_PATCH "${libaec_VERSION_PATCH}") -set(CPACK_PACKAGE_INSTALL_DIRECTORY "libaec") -set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/Copyright.txt") - -if(WIN32) - set(CPACK_GENERATOR "WIX") - set(CPACK_WIX_UPGRADE_GUID "E58A21F6-BB99-46B5-8AD8-6114E4CCA0A6") - if(CMAKE_CL_64) - set(CPACK_INSTALL_DIRECTORY "$PROGRAMFILES64") - set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY - "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION} (Win64)") - else(CMAKE_CL_64) - set(CPACK_INSTALL_DIRECTORY "$PROGRAMFILES") - set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY - "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}") - endif(CMAKE_CL_64) -endif(WIN32) - -include(CPack) diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Copyright.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Copyright.txt deleted file mode 100644 index 72969b0..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Copyright.txt +++ /dev/null @@ -1,37 +0,0 @@ -Copyright 2012 - 2017 - -Mathis Rosenhauer, Moritz Hanke, Joerg Behrens -Deutsches Klimarechenzentrum GmbH -Bundesstr. 45a -20146 Hamburg -Germany - -Luis Kornblueh -Max-Planck-Institut fuer Meteorologie -Bundesstr. 53 -20146 Hamburg -Germany - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/INSTALL b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/INSTALL deleted file mode 100644 index 85c67d5..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/INSTALL +++ /dev/null @@ -1,95 +0,0 @@ -Getting libaec -============== - -The source code of libaec is hosted at DKRZ GitLab. - -## Source code and binary releases - -The latest releases of libaec can be downloaded at the following -location: - - https://gitlab.dkrz.de/k202009/libaec/tags - -## Developer snapshot - - git clone https://gitlab.dkrz.de/k202009/libaec.git - - -Installation -============ - -## General considerations - -Libaec achieves the best performance on 64 bit systems. The library -will work correctly on 32 bit systems but encoding and decoding -performance will be much lower. - -## Installation from source code release with configure - -The most common installation procedure on Unix-like systems looks as -follows: - -Unpack the tar archive and change into the unpacked directory. - - mkdir build - cd build - ../configure - make check install - -## Installation from source code release with CMake - -As an alternative, you can use CMake to install libaec. - -Unpack the tar archive and change into the unpacked directory. - - mkdir build - cd build - cmake .. - make install - -You can set options for compiling using the CMake GUI by replacing the cmake -command with - - cmake-gui .. - -or by setting the options manually, e.g. - - cmake -DCMAKE_INSTALL_PREFIX=~/local .. - -in order to set the install prefix to ~/local - -CMake can also generate project files for Microsoft Visual Studio when -used in Windows. - -## Installation from cloned repository - -The configure script is not included in the repository. You can -generate it with autotools: - - cd libaec - autoreconv -iv - mkdir build - cd build - ../configure - -Also not included are CCSDS sample data which are needed for -testing. They have to be downloaded prior to running 'make check': - - make update-sampledata - make check install - - -Intel compiler settings -======================= - -The Intel compiler can improve performance by vectorizing certain -parts of the code on x86 architectures. Assuming your CPU supports -AVX2, the following options will increase encoding speed. - - ../configure CC=icc - make CFLAGS="-O3 -xCORE-AVX2" bench - -On a 3.4 GHz E3-1240 v3 we see more than 400 MiB/s for encoding -typical data. - -Using other SIMD instruction sets on older CPUs may also help. diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Makefile.am b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Makefile.am deleted file mode 100644 index c262e41..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/Makefile.am +++ /dev/null @@ -1,15 +0,0 @@ -ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src tests -EXTRA_DIST = doc/patent.txt CMakeLists.txt cmake/config.h.in \ -cmake/macros.cmake README.md README.SZIP CHANGELOG.md Copyright.txt data - -sampledata = 121B2TestData -sampledata_url = https://cwe.ccsds.org/sls/docs/SLS-DC/BB121B2TestData/$(sampledata).zip -update-sampledata: - cd $(srcdir)/data && \ - rm -rf $(sampledata) && \ - curl $(sampledata_url) -O && unzip $(sampledata).zip && \ - rm -f $(sampledata).zip && \ - cd $(sampledata)/AllOptions && mv -f test_P512n22.dat test_p512n22.dat - -.PHONY: update-sampledata diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.SZIP b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.SZIP deleted file mode 100644 index c62b3b6..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.SZIP +++ /dev/null @@ -1,15 +0,0 @@ -********************************************************************** - SZIP compatibility -********************************************************************** - -Libaec includes a free drop-in replacement for the SZIP -library[1]. Just replace SZIP's shared library libsz.so* with -libaec.so* and libsz.so* from libaec. For Windows the DLLs are called -SZIP.DLL and AEC.DLL. Code which is dynamically linked with SZIP such -as HDF5 should continue to work with libaec. No re-compilation -required. - -HDF5 files which contain SZIP encoded data can be decoded by HDF5 -using libaec and vice versa. - -[1] http://www.hdfgroup.org/doc_resource/SZIP/ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.md b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.md deleted file mode 100644 index 6b0817e..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/README.md +++ /dev/null @@ -1,225 +0,0 @@ -# libaec - Adaptive Entropy Coding library - -Libaec provides fast lossless compression of 1 up to 32 bit wide -signed or unsigned integers (samples). The library achieves best -results for low entropy data as often encountered in space imaging -instrument data or numerical model output from weather or climate -simulations. While floating point representations are not directly -supported, they can also be efficiently coded by grouping exponents -and mantissa. - -Libaec implements -[Golomb-Rice](http://en.wikipedia.org/wiki/Golomb_coding) coding as -defined in the Space Data System Standard documents [121.0-B-2][1] and -[120.0-G-2][2]. - -## Downloads - -Source code and binary installer can be [downloaded here](https://gitlab.dkrz.de/k202009/libaec/tags). - -## Patent - -In [patent.txt](doc/patent.txt) a statement on potentially -applying intellectual property rights is given. - -## Installation - -See [INSTALL](INSTALL) for details. - -## SZIP Compatibility - -[Libaec can replace SZIP](README.SZIP). - -## Encoding - -In this context efficiency refers to the size of the encoded -data. Performance refers to the time it takes to encode data. - -Suppose you have an array of 32 bit signed integers you want to -compress. The pointer pointing to the data shall be called `*source`, -output goes into `*dest`. - -```c -#include - -... - struct aec_stream strm; - int32_t *source; - unsigned char *dest; - - /* input data is 32 bits wide */ - strm.bits_per_sample = 32; - - /* define a block size of 16 */ - strm.block_size = 16; - - /* the reference sample interval is set to 128 blocks */ - strm.rsi = 128; - - /* input data is signed and needs to be preprocessed */ - strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS; - - /* pointer to input */ - strm.next_in = (unsigned char *)source; - - /* length of input in bytes */ - strm.avail_in = source_length * sizeof(int32_t); - - /* pointer to output buffer */ - strm.next_out = dest; - - /* length of output buffer in bytes */ - strm.avail_out = dest_length; - - /* initialize encoding */ - if (aec_encode_init(&strm) != AEC_OK) - return 1; - - /* Perform encoding in one call and flush output. */ - /* In this example you must be sure that the output */ - /* buffer is large enough for all compressed output */ - if (aec_encode(&strm, AEC_FLUSH) != AEC_OK) - return 1; - - /* free all resources used by encoder */ - aec_encode_end(&strm); -... -``` - -`block_size` can vary from 8 to 64 samples. Smaller blocks allow the -compression to adapt more rapidly to changing source -statistics. Larger blocks create less overhead but can be less -efficient if source statistics change across the block. - -`rsi` sets the reference sample interval. A large RSI will improve -performance and efficiency. It will also increase memory requirements -since internal buffering is based on RSI size. A smaller RSI may be -desirable in situations where each RSI will be packetized and possible -error propagation has to be minimized. - -### Flags: - -* `AEC_DATA_SIGNED`: input data are signed integers. Specifying this - correctly increases compression efficiency. Default is unsigned. - -* `AEC_DATA_PREPROCESS`: preprocessing input will improve compression - efficiency if data samples are correlated. It will only cost - performance for no gain in efficiency if the data is already - uncorrelated. - -* `AEC_DATA_MSB`: input data is stored most significant byte first - i.e. big endian. You have to specify `AEC_DATA_MSB` even if your host - architecture is big endian. Default is little endian on all - architectures. - -* `AEC_DATA_3BYTE`: the 17 to 24 bit input data is stored in three - bytes. This flag has no effect for other sample sizes. - -* `AEC_RESTRICTED`: use a restricted set of code options. This option is - only valid for `bits_per_sample` <= 4. - -* `AEC_PAD_RSI`: assume that the encoded RSI is padded to the next byte - boundary while decoding. The preprocessor macro `ENABLE_RSI_PADDING` - needs to be defined while compiling for the encoder to honour this - flag. - -### Data size: - -The following rules apply for deducing storage size from sample size -(`bits_per_sample`): - - **sample size** | **storage size** ---- | --- - 1 - 8 bits | 1 byte - 9 - 16 bits | 2 bytes -17 - 24 bits | 3 bytes (only if `AEC_DATA_3BYTE` is set) -25 - 32 bits | 4 bytes (if `AEC_DATA_3BYTE` is set) -17 - 32 bits | 4 bytes (if `AEC_DATA_3BYTE` is not set) - -If a sample requires less bits than the storage size provides, then -you have to make sure that unused bits are not set. Libaec does not -check this for performance reasons and will produce undefined output -if unused bits are set. All input data must be a multiple of the -storage size in bytes. Remaining bytes which do not form a complete -sample will be ignored. - -Libaec accesses `next_in` and `next_out` buffers only bytewise. There -are no alignment requirements for these buffers. - -### Flushing: - -`aec_encode` can be used in a streaming fashion by chunking input and -output and specifying `AEC_NO_FLUSH`. The function will return if either -the input runs empty or the output buffer is full. The calling -function can check `avail_in` and `avail_out` to see what occurred. The -last call to `aec_encode()` must set `AEC_FLUSH` to drain all -output. [aec.c](src/aec.c) is an example of streaming usage of encoding and -decoding. - -### Output: - -Encoded data will be written to the buffer submitted with -`next_out`. The length of the compressed data is `total_out`. - -See libaec.h for a detailed description of all relevant structure -members and constants. - - -## Decoding - -Using decoding is very similar to encoding, only the meaning of input -and output is reversed. - -```c -#include - -... - struct aec_stream strm; - /* this is now the compressed data */ - unsigned char *source; - /* here goes the uncompressed result */ - int32_t *dest; - - strm.bits_per_sample = 32; - strm.block_size = 16; - strm.rsi = 128; - strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS; - strm.next_in = source; - strm.avail_in = source_length; - strm.next_out = (unsigned char *)dest; - strm.avail_out = dest_lenth * sizeof(int32_t); - if (aec_decode_init(&strm) != AEC_OK) - return 1; - if (aec_decode(&strm, AEC_FLUSH) != AEC_OK) - return 1; - aec_decode_end(&strm); -... -``` - -It is strongly recommended that the size of the output buffer -(`next_out`) is a multiple of the storage size in bytes. If the buffer -is not a multiple of the storage size and the buffer gets filled to -the last sample, the error code `AEC_MEM_ERROR` is returned. - -It is essential for decoding that parameters like `bits_per_sample`, -`block_size`, `rsi`, and `flags` are exactly the same as they were for -encoding. Libaec does not store these parameters in the coded stream -so it is up to the calling program to keep the correct parameters -between encoding and decoding. - -The actual values of coding parameters are in fact only relevant for -efficiency and performance. Data integrity only depends on consistency -of the parameters. - - -## References - -[Consultative Committee for Space Data Systems. Lossless Data -Compression. Recommendation for Space Data System Standards, CCSDS -121.0-B-2. Blue Book. Issue 2. Washington, D.C.: CCSDS, May 2012.][1] -[1]: http://public.ccsds.org/publications/archive/121x0b2.pdf - -[Consultative Committee for Space Data Systems. Lossless Data -Compression. Recommendation for Space Data System Standards, CCSDS -120.0-G-3. Green Book. Issue 3. Washington, D.C.: CCSDS, April 2013.][2] -[2]: http://public.ccsds.org/publications/archive/120x0g3.pdf diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/THANKS b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/THANKS deleted file mode 100644 index 8281b0c..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/THANKS +++ /dev/null @@ -1,16 +0,0 @@ -********************************************************************** - CCSDS sample data for libaec testing -********************************************************************** - -Thanks to the chairman of CCSDS SLS-MHDC, Aaron Kiely, to let us -distribute BB121B2 test data with libaec. The data files are used -during 'make check'. - -The most recent version of the data is available from the CCSDS web -site: - -http://cwe.ccsds.org/sls/docs/Forms/AllItems.aspx?RootFolder=%2Fsls%2Fdocs%2FSLS%2DDC%2FBB121B2TestData& - -You can also update the data included in the distribution with - - make update-sampledata diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/config.h.in b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/config.h.in deleted file mode 100644 index d5783e0..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/config.h.in +++ /dev/null @@ -1,8 +0,0 @@ -#cmakedefine HAVE_MALLOC_H 1 -#cmakedefine HAVE_STDINT_H 1 -#cmakedefine WORDS_BIGENDIAN 1 -#cmakedefine HAVE_DECL___BUILTIN_CLZLL 1 -#cmakedefine HAVE_BSR64 1 -#cmakedefine HAVE_SNPRINTF 1 -#cmakedefine HAVE__SNPRINTF 1 -#cmakedefine HAVE__SNPRINTF_S 1 diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/macros.cmake b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/macros.cmake deleted file mode 100644 index a568b15..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/cmake/macros.cmake +++ /dev/null @@ -1,77 +0,0 @@ -macro(check_clzll VARIABLE) - check_c_source_compiles( - "int main(int argc, char *argv[]) -{return __builtin_clzll(1LL);}" - ${VARIABLE} - ) -endmacro() - -macro(check_bsr64 VARIABLE) - check_c_source_compiles( - "int main(int argc, char *argv[]) -{unsigned long foo; unsigned __int64 bar=1LL; -return _BitScanReverse64(&foo, bar);}" - ${VARIABLE} - ) -endmacro() - -macro(find_inline_keyword) - #Inspired from http://www.cmake.org/Wiki/CMakeTestInline - set(INLINE_TEST_SRC "/* Inspired by autoconf's c.m4 */ -static inline int static_foo(){return 0\;} -int main(int argc, char *argv[]){return 0\;} -") - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCInline.c - ${INLINE_TEST_SRC}) - - foreach(KEYWORD "inline" "__inline__" "__inline") - if(NOT DEFINED C_INLINE) - try_compile(C_HAS_${KEYWORD} - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCInline.c - COMPILE_DEFINITIONS "-Dinline=${KEYWORD}" - ) - if(C_HAS_${KEYWORD}) - set(C_INLINE TRUE) - add_definitions("-Dinline=${KEYWORD}") - message(STATUS "Inline keyword found - ${KEYWORD}") - endif(C_HAS_${KEYWORD}) - endif(NOT DEFINED C_INLINE) - endforeach(KEYWORD) - - if(NOT DEFINED C_INLINE) - add_definitions("-Dinline=") - message(STATUS "Inline keyword - not found") - endif(NOT DEFINED C_INLINE) -endmacro(find_inline_keyword) - -macro(find_restrict_keyword) - set(RESTRICT_TEST_SRC "/* Inspired by autoconf's c.m4 */ -int foo (int * restrict ip){return ip[0]\;} -int main(int argc, char *argv[]){int s[1]\; -int * restrict t = s\; t[0] = 0\; return foo(t)\;} -") - - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCRestrict.c - ${RESTRICT_TEST_SRC}) - - foreach(KEYWORD "restrict" "__restrict" "__restrict__" "_Restrict") - if(NOT DEFINED C_RESTRICT) - try_compile(C_HAS_${KEYWORD} - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/CMakeTestCRestrict.c - COMPILE_DEFINITIONS "-Drestrict=${KEYWORD}" - ) - if(C_HAS_${KEYWORD}) - set(C_RESTRICT TRUE) - add_definitions("-Drestrict=${KEYWORD}") - message(STATUS "Restrict keyword found - ${KEYWORD}") - endif(C_HAS_${KEYWORD}) - endif(NOT DEFINED C_RESTRICT) - endforeach(KEYWORD) - - if(NOT DEFINED C_RESTRICT) - add_definitions("-Drestrict=") - message(STATUS "Restrict keyword - not found") - endif(NOT DEFINED C_RESTRICT) -endmacro(find_restrict_keyword) diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/configure.ac b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/configure.ac deleted file mode 100644 index e958719..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/configure.ac +++ /dev/null @@ -1,34 +0,0 @@ -AC_PREREQ([2.64]) -AC_INIT([libaec], [1.0.2], [rosenhauer@dkrz.de]) - -AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_SRCDIR([src/libaec.h]) -AC_CONFIG_HEADERS(config/config.h) - -LT_INIT -gl_VISIBILITY -AM_INIT_AUTOMAKE([foreign]) -AM_MAINTAINER_MODE([enable]) - -AC_PROG_CC_C99 - -AC_HEADER_STDC - -AC_C_BIGENDIAN -AC_C_INLINE -AC_TYPE_INT64_T -AC_TYPE_SIZE_T -AC_TYPE_UINT32_T -AC_TYPE_UINT64_T -AC_TYPE_UINT8_T -AC_C_INLINE -AC_C_RESTRICT - -AC_CHECK_FUNCS([memset strstr snprintf]) -AC_CHECK_DECLS(__builtin_clzll) - -AM_EXTRA_RECURSIVE_TARGETS([bench benc bdec]) - -AC_CONFIG_FILES([Makefile src/Makefile tests/Makefile]) -AC_OUTPUT diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/data/typical.rz b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/data/typical.rz deleted file mode 100644 index fc1e827..0000000 Binary files a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/data/typical.rz and /dev/null differ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/README b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/README deleted file mode 100644 index 4a96d8e..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/README +++ /dev/null @@ -1,13 +0,0 @@ ---------------------- - 32 bits vs. 64 bits ---------------------- - -libaec compiles and runs both in 32 and 64 bits. For encoding and -decoding I see a performance degradation with 32 bit binaries of 38% -and 33% resp. - -The main reason for this is that encoding and decoding use a buffer of -type uin64_t to align a potentially 32 bit wide sample with the output -(encoding) or input (decoding) stream. Avoiding the use of this -uin64_t buffer variable would degrade 64 bit performance for a modest -gain in 32 bit performance so I decided against it. diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/logo.svg b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/logo.svg deleted file mode 100644 index ebe4e12..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/logo.svg +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - C - AE - - Golomb Rice Adaptive Entropy Coder - - - diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/patent.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/patent.txt deleted file mode 100644 index c49e4bb..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/doc/patent.txt +++ /dev/null @@ -1,57 +0,0 @@ -------------------------------------------------------------------- -Lossless Data Compression, Recommended Standard, Issue 2, May 2012, -CCSDS 121.0-B-2 -------------------------------------------------------------------- -Patent Considerations, Section A3, page 37 - -With respect to this issue we got the following information from -NASA's Chief Patent Counsel: - -From: Geurts, Bryan A. (GSFC-1401) -Sent: Tuesday, April 21, 2009 12:14 PM -To: Yeh, Pen-Shu (GSFC-5670) -Subject: RE: Need help from you again - -Pen-Shu, - -The license agreement with ICs, LLC, for Patent Numbers US 5,448,642, -US 5,687,255 and US 5,822,457, has been terminated effective March 31, -2009. NASA has no other licensee of these patents. It is NASA's intent -to not pay the remaining maintenance fees owing on these patents, -thereby allowing them to eventually become abandoned, and to not -enforce the patents. - -If you have any questions or concerns, please do not hesitate to contact me. - -Bryan A. Geurts -Chief Patent Counsel -NASA Goddard Space Flight Center -Code 140.1, 8800 Greenbelt Road -Greenbelt, MD 20771 -Phone: (301) 286-7352 -Fax: (301) 286-9502 - -Reassuring this information in August 2012 because of the patent -considerations in the above referenced CCSDS standard: - -From: Geurts, Bryan A. (GSFC-1401) -Sent: Friday, August 31, 2012 10:36 AM -To: Luis Kornblueh -Subject: RE: Patent 5448642 - -Dear Mr. Kornblueh - -Per my email cited above, NASA has no interest in licensing or -maintaining US Patent No. 5448642 and therefore has no objection to -your use of its teachings and claims. However, while NASA has -effectively abandoned this patent, any use of the subject matter is at -your own risk and NASA cannot warrant that its use will satisfy your -particular needs or that your use will not infringe on another's -intellectual property rights. - -------------------------------------------------------------------- - -The cited CCSDS standard does not refer to any other intellectual -property rights. - -------------------------------------------------------------------- diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/CMakeLists.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/CMakeLists.txt deleted file mode 100644 index b57e642..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") - message(FATAL_ERROR "You need to build with Clang for fuzzing to work") -endif() - -add_library(aec_fuzz STATIC ${libaec_SRCS}) -add_executable(fuzz_target fuzz_target.cc) - -set(FUZZ_TARGET_LINK_FLAGS "-fsanitize=address") -set(FUZZ_TARGET_COMPILE_FLAGS - "-g" - "-fsanitize=address" - "-fsanitize-coverage=trace-pc-guard,indirect-calls,edge") -set_target_properties(fuzz_target aec_fuzz PROPERTIES - LINK_FLAGS ${FUZZ_TARGET_LINK_FLAGS} - COMPILE_OPTIONS "${FUZZ_TARGET_COMPILE_FLAGS}") - -target_link_libraries(fuzz_target aec_fuzz Fuzzer) diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/fuzz_target.cc b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/fuzz_target.cc deleted file mode 100644 index 63fcf79..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/fuzzing/fuzz_target.cc +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include "libaec.h" - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { - struct aec_stream strm; - int32_t *source; - unsigned char *dest; - - if (Size < 2) - return 0; - - dest = (unsigned char *)malloc(Size * 4); - strm.bits_per_sample = (Data[0] & 0x1f) | 1; - strm.block_size = 8 << (Data[1] & 3); - strm.rsi = 2; - strm.flags = AEC_DATA_PREPROCESS; - if (Data[1] & 0x80) - strm.flags |= AEC_DATA_MSB; - if (Data[1] & 0x40) - strm.flags |= AEC_DATA_SIGNED; - if (strm.bits_per_sample <= 24 && - strm.bits_per_sample > 16 && - Data[1] & 0x10) - strm.flags |= AEC_DATA_3BYTE; - strm.next_in = (unsigned char *)(Data + 2); - strm.avail_in = Size - 2; - strm.next_out = dest; - strm.avail_out = (Size - 2) * 4; - if (Data[1] & 0x20) - aec_buffer_encode(&strm); - else - aec_buffer_decode(&strm); - free(dest); - return 0; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/libaec.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/libaec.h deleted file mode 120000 index ccf5359..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/libaec.h +++ /dev/null @@ -1 +0,0 @@ -../src/libaec.h \ No newline at end of file diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/szlib.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/szlib.h deleted file mode 120000 index f74bf00..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/include/szlib.h +++ /dev/null @@ -1 +0,0 @@ -../src/szlib.h \ No newline at end of file diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/CMakeLists.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/CMakeLists.txt deleted file mode 100644 index 93c681b..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -add_library(aec ${LIB_TYPE} ${libaec_SRCS}) -set_target_properties(aec PROPERTIES VERSION 0.0.8 SOVERSION 0) -add_library(sz ${LIB_TYPE} sz_compat.c) -set_target_properties(sz PROPERTIES VERSION 2.0.1 SOVERSION 2) - -target_link_libraries(sz aec) -if(WIN32 AND BUILD_SHARED_LIBS) - set_target_properties (aec PROPERTIES DEFINE_SYMBOL "BUILDING_LIBAEC") - set_target_properties (sz PROPERTIES DEFINE_SYMBOL "BUILDING_LIBAEC") - set_target_properties(sz PROPERTIES OUTPUT_NAME "szip") -endif(WIN32 AND BUILD_SHARED_LIBS) - -add_executable(aec_client aec.c) -set_target_properties(aec_client PROPERTIES OUTPUT_NAME "aec") -target_link_libraries(aec_client aec) - -if(UNIX) - add_executable(utime EXCLUDE_FROM_ALL utime.c) - add_custom_target(bench - COMMAND ${CMAKE_SOURCE_DIR}/src/benc.sh ${CMAKE_SOURCE_DIR}/data/typical.rz - COMMAND ${CMAKE_SOURCE_DIR}/src/bdec.sh - DEPENDS aec_client utime) -endif(UNIX) - -include(GNUInstallDirs) -if(WIN32) - install(TARGETS aec sz - RUNTIME - DESTINATION bin - ARCHIVE - DESTINATION lib - COMPONENT libraries) -else(WIN32) - install(TARGETS aec sz - LIBRARY - DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE - DESTINATION ${CMAKE_INSTALL_LIBDIR} - COMPONENT libraries) - install(FILES aec.1 - DESTINATION ${CMAKE_INSTALL_FULL_MANDIR}/man1 COMPONENT doc) -endif(WIN32) - -install(FILES libaec.h szlib.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT headers) -install(TARGETS aec_client - RUNTIME - DESTINATION ${CMAKE_INSTALL_BINDIR} - COMPONENT applications) - -set(CPACK_COMPONENTS_ALL applications libraries headers doc) diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/Makefile.am b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/Makefile.am deleted file mode 100644 index de81372..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/Makefile.am +++ /dev/null @@ -1,28 +0,0 @@ -AM_CFLAGS = $(CFLAG_VISIBILITY) -AM_CPPFLAGS = -DBUILDING_LIBAEC -lib_LTLIBRARIES = libaec.la libsz.la -libaec_la_SOURCES = encode.c encode_accessors.c decode.c \ -encode.h encode_accessors.h decode.h -libaec_la_LDFLAGS = -version-info 0:8:0 -no-undefined - -libsz_la_SOURCES = sz_compat.c -libsz_la_LIBADD = libaec.la -libsz_la_LDFLAGS = -version-info 2:1:0 -no-undefined - -include_HEADERS = libaec.h szlib.h - -bin_PROGRAMS = aec -noinst_PROGRAMS = utime -utime_SOURCES = utime.c -aec_LDADD = libaec.la -aec_SOURCES = aec.c -dist_man_MANS = aec.1 - -EXTRA_DIST = CMakeLists.txt benc.sh bdec.sh -CLEANFILES = bench.dat bench.rz - -bench-local: all benc bdec -benc-local: all - $(srcdir)/benc.sh $(top_srcdir)/data/typical.rz -bdec-local: all - top_srcdir=$(top_srcdir) $(srcdir)/bdec.sh diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.1 b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.1 deleted file mode 100644 index fa7dc1f..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.1 +++ /dev/null @@ -1,57 +0,0 @@ -.TH AEC 1 -.SH NAME -aec \- compress or expand files -.SH SYNOPSIS -.B aec -[\fB\-3\fR] -[\fB\-b\fR \fIBYTES\fR] -[\fB\-d\fR] -[\fB\-j\fR \fISAMPLES\fR] -[\fB\-m\fR] -[\fB\-n\fR \fIBITS\fR] -[\fB\-N\fR] -[\fB\-p\fR] -[\fB\-r\fR \fIBLOCKS\fR] -[\fB\-s\fR] -[\fB\-t\fR] -.IR infile -.IR outfile -.SH DESCRIPTION -.IR Aec -performs lossless compression and decompression with Golomb-Rice coding -as defined in the Space Data System Standard documents 121.0-B-2. -.SH OPTIONS -.TP -\fB \-3\fR -24 bit samples are stored in 3 bytes -.TP -\fB \-b\fR\ \fI\,BYTES\fR -internal buffer size in bytes -.TP -\fB \-d\fR -decompress \fIinfile\fR; if option \-d is not used then compress -\fIinfile\fR -.TP -\fB \-j\fR \fI\,SAMPLES\fR -block size in samples -.TP -\fB \-m\fR -samples are MSB first; default is LSB first -.TP -\fB \-n\fR \fI\,BITS\fR -bits per sample -.TP -\fB \-N\fR -disable pre/post processing -.TP -\fB \-p\fR -pad RSI to byte boundary -.TP -\fB \-r\fR \fI\,BLOCKS\fR -reference sample interval in blocks -.TP -\fB \-s\fR -samples are signed; default is unsigned -.TP -\fB \-t\fR -use restricted set of code options diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.c deleted file mode 100644 index b3c6a36..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/aec.c +++ /dev/null @@ -1,258 +0,0 @@ -/** - * @file aec.c - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * CLI frontend for Adaptive Entropy Coding library - * - */ - -#include -#include -#include - -#include "libaec.h" - -#define CHUNK 10485760 - -int get_param(unsigned int *param, int *iarg, char *argv[]) -{ - if (strlen(argv[*iarg]) == 2) { - (*iarg)++; - if (argv[*iarg][0] == '-') - return 1; - else - *param = atoi(argv[*iarg]); - } else { - *param = atoi(&argv[*iarg][2]); - } - return 0; -} - -int main(int argc, char *argv[]) -{ - struct aec_stream strm; - unsigned char *in; - unsigned char *out; - size_t total_out; - unsigned int chunk; - int status; - int input_avail, output_avail; - char *infn, *outfn; - FILE *infp, *outfp; - int dflag; - char *opt; - int iarg; - - chunk = CHUNK; - strm.bits_per_sample = 8; - strm.block_size = 8; - strm.rsi = 2; - strm.flags = AEC_DATA_PREPROCESS; - dflag = 0; - iarg = 1; - - while (iarg < argc - 2) { - opt = argv[iarg]; - if (opt[0] != '-') - goto FAIL; - switch (opt[1]) { - case '3': - strm.flags |= AEC_DATA_3BYTE; - break; - case 'N': - strm.flags &= ~AEC_DATA_PREPROCESS; - break; - case 'b': - if (get_param(&chunk, &iarg, argv)) - goto FAIL; - break; - case 'd': - dflag = 1; - break; - case 'j': - if (get_param(&strm.block_size, &iarg, argv)) - goto FAIL; - break; - case 'm': - strm.flags |= AEC_DATA_MSB; - break; - case 'n': - if (get_param(&strm.bits_per_sample, &iarg, argv)) - goto FAIL; - break; - case 'p': - strm.flags |= AEC_PAD_RSI; - break; - case 'r': - if (get_param(&strm.rsi, &iarg, argv)) - goto FAIL; - break; - case 's': - strm.flags |= AEC_DATA_SIGNED; - break; - case 't': - strm.flags |= AEC_RESTRICTED; - break; - default: - goto FAIL; - } - iarg++; - } - - if (argc - iarg < 2) - goto FAIL; - - infn = argv[iarg]; - outfn = argv[iarg + 1]; - - if (strm.bits_per_sample > 16) { - if (strm.bits_per_sample <= 24 && strm.flags & AEC_DATA_3BYTE) - chunk *= 3; - else - chunk *= 4; - } else if (strm.bits_per_sample > 8) { - chunk *= 2; - } - - out = (unsigned char *)malloc(chunk); - in = (unsigned char *)malloc(chunk); - - if (in == NULL || out == NULL) - exit(-1); - - total_out = 0; - strm.avail_in = 0; - strm.avail_out = chunk; - strm.next_out = out; - - input_avail = 1; - output_avail = 1; - - if ((infp = fopen(infn, "rb")) == NULL) { - fprintf(stderr, "ERROR: cannot open input file %s\n", infn); - return 1; - } - if ((outfp = fopen(outfn, "wb")) == NULL) { - fprintf(stderr, "ERROR: cannot open output file %s\n", infn); - return 1; - } - - if (dflag) - status = aec_decode_init(&strm); - else - status = aec_encode_init(&strm); - - if (status != AEC_OK) { - fprintf(stderr, "ERROR: initialization failed (%d)\n", status); - return 1; - } - - while(input_avail || output_avail) { - if (strm.avail_in == 0 && input_avail) { - strm.avail_in = fread(in, 1, chunk, infp); - if (strm.avail_in != chunk) - input_avail = 0; - strm.next_in = in; - } - - if (dflag) - status = aec_decode(&strm, AEC_NO_FLUSH); - else - status = aec_encode(&strm, AEC_NO_FLUSH); - - if (status != AEC_OK) { - fprintf(stderr, "ERROR: %i\n", status); - return 1; - } - - if (strm.total_out - total_out > 0) { - fwrite(out, strm.total_out - total_out, 1, outfp); - total_out = strm.total_out; - output_avail = 1; - strm.next_out = out; - strm.avail_out = chunk; - } else { - output_avail = 0; - } - - } - - if (dflag) { - aec_decode_end(&strm); - } else { - if ((status = aec_encode(&strm, AEC_FLUSH)) != AEC_OK) { - fprintf(stderr, "ERROR: while flushing output (%i)\n", status); - return 1; - } - - if (strm.total_out - total_out > 0) - fwrite(out, strm.total_out - total_out, 1, outfp); - - aec_encode_end(&strm); - } - - fclose(infp); - fclose(outfp); - free(in); - free(out); - return 0; - -FAIL: - fprintf(stderr, "NAME\n\taec - encode or decode files "); - fprintf(stderr, "with Adaptive Entropy Coding\n\n"); - fprintf(stderr, "SYNOPSIS\n\taec [OPTION]... SOURCE DEST\n"); - fprintf(stderr, "\nOPTIONS\n"); - fprintf(stderr, "\t-3\n\t\t24 bit samples are stored in 3 bytes\n"); - fprintf(stderr, "\t-N\n\t\tdisable pre/post processing\n"); - fprintf(stderr, "\t-b size\n\t\tinternal buffer size in bytes\n"); - fprintf(stderr, "\t-d\n\t\tdecode SOURCE. If -d is not used: encode.\n"); - fprintf(stderr, "\t-j samples\n\t\tblock size in samples\n"); - fprintf(stderr, "\t-m\n\t\tsamples are MSB first. Default is LSB\n"); - fprintf(stderr, "\t-n bits\n\t\tbits per sample\n"); - fprintf(stderr, "\t-p\n\t\tpad RSI to byte boundary\n"); - fprintf(stderr, "\t-r blocks\n\t\treference sample interval in blocks\n"); - fprintf(stderr, "\t-s\n\t\tsamples are signed. Default is unsigned\n"); - fprintf(stderr, "\t-t\n\t\tuse restricted set of code options\n\n"); - return 1; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/bdec.sh b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/bdec.sh deleted file mode 100755 index 5b473d8..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/bdec.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -e -if [ ! -f bench.rz ]; then - echo "No encoded file found. Encoding now..." - ${top_srcdir}/src/benc.sh ${top_srcdir}/data/typical.rz -fi -rm -f dec.dat -bsize=$(wc -c bench.dat | awk '{print $1}') -utime=$(./utime ./aec -d -n16 -j64 -r256 -m bench.rz dec.dat 2>&1) -perf=$(awk "BEGIN {print ${bsize}/1048576/${utime}}") -echo "*** Decoding with $perf MiB/s user time ***" -cmp bench.dat dec.dat -rm -f dec.dat diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/benc.sh b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/benc.sh deleted file mode 100755 index 5bc031c..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/benc.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -set -e -TEST_DATA=$1 -AEC=./aec -if [ ! -f bench.dat ]; then - rm -f typical.dat - $AEC -d -n16 -j64 -r256 -m $TEST_DATA typical.dat - for i in $(seq 0 499); - do - cat typical.dat >> bench.dat - done - rm -f typical.dat -fi -rm -f bench.rz -utime=$(./utime $AEC -n16 -j64 -r256 -m bench.dat bench.rz 2>&1) -echo $utime -bsize=$(wc -c bench.dat | awk '{print $1}') -perf=$(awk "BEGIN {print ${bsize}/1048576/${utime}}") -echo "*** Encoding with $perf MiB/s user time ***" diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.c deleted file mode 100644 index 160d4c3..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.c +++ /dev/null @@ -1,842 +0,0 @@ -/** - * @file decode.c - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Adaptive Entropy Decoder - * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 - * - */ - -#include -#include -#include - -#include "decode.h" -#include "libaec.h" - -#if HAVE_BSR64 -# include -#endif - -#define ROS 5 -#define RSI_USED_SIZE(state) ((size_t)(state->rsip - state->rsi_buffer)) -#define BUFFERSPACE(strm) (strm->avail_in >= strm->state->in_blklen \ - && strm->avail_out >= strm->state->out_blklen) - -#define FLUSH(KIND) \ - static void flush_##KIND(struct aec_stream *strm) \ - { \ - uint32_t *flush_end, *bp, half_d; \ - int32_t data, m; \ - struct internal_state *state = strm->state; \ - \ - flush_end = state->rsip; \ - if (state->pp) { \ - if (state->flush_start == state->rsi_buffer \ - && state->rsip > state->rsi_buffer) { \ - state->last_out = *state->rsi_buffer; \ - \ - if (strm->flags & AEC_DATA_SIGNED) { \ - m = UINT32_C(1) << (strm->bits_per_sample - 1); \ - /* Reference samples have to be sign extended */ \ - state->last_out = (state->last_out ^ m) - m; \ - } \ - put_##KIND(strm, (uint32_t)state->last_out); \ - state->flush_start++; \ - } \ - \ - data = state->last_out; \ - \ - if (state->xmin == 0) { \ - uint32_t xmax, med, d; \ - med = state->xmax / 2 + 1; \ - xmax = state->xmax; \ - \ - for (bp = state->flush_start; bp < flush_end; bp++) { \ - uint32_t mask; \ - d = *bp; \ - half_d = (d >> 1) + (d & 1); \ - /*in this case: data >= med == data & med */ \ - mask = (data & med)?xmax:0; \ - \ - /*in this case: xmax - data == xmax ^ data */ \ - if (half_d <= (mask ^ (uint32_t)data)) { \ - data += (d >> 1)^(~((d & 1) - 1)); \ - } else { \ - data = mask ^ d; \ - } \ - put_##KIND(strm, (uint32_t)data); \ - } \ - state->last_out = data; \ - } else { \ - int32_t xmax, d; \ - xmax = state->xmax; \ - \ - for (bp = state->flush_start; bp < flush_end; bp++) { \ - d = *bp; \ - half_d = ((uint32_t)d >> 1) + (d & 1); \ - \ - if (data < 0) { \ - if (half_d <= xmax + (uint32_t)data + 1) { \ - data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \ - } else { \ - data = d - xmax - 1; \ - } \ - } else { \ - if (half_d <= xmax - (uint32_t)data) { \ - data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \ - } else { \ - data = xmax - d; \ - } \ - } \ - put_##KIND(strm, (uint32_t)data); \ - } \ - state->last_out = data; \ - } \ - } else { \ - for (bp = state->flush_start; bp < flush_end; bp++) \ - put_##KIND(strm, *bp); \ - } \ - state->flush_start = state->rsip; \ - } - - -static inline void put_msb_32(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)(data >> 24); - *strm->next_out++ = (unsigned char)(data >> 16); - *strm->next_out++ = (unsigned char)(data >> 8); - *strm->next_out++ = (unsigned char)data; -} - -static inline void put_msb_24(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)(data >> 16); - *strm->next_out++ = (unsigned char)(data >> 8); - *strm->next_out++ = (unsigned char)data; -} - -static inline void put_msb_16(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)(data >> 8); - *strm->next_out++ = (unsigned char)data; -} - -static inline void put_lsb_32(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)data; - *strm->next_out++ = (unsigned char)(data >> 8); - *strm->next_out++ = (unsigned char)(data >> 16); - *strm->next_out++ = (unsigned char)(data >> 24); -} - -static inline void put_lsb_24(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)data; - *strm->next_out++ = (unsigned char)(data >> 8); - *strm->next_out++ = (unsigned char)(data >> 16); -} - -static inline void put_lsb_16(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)data; - *strm->next_out++ = (unsigned char)(data >> 8); -} - -static inline void put_8(struct aec_stream *strm, uint32_t data) -{ - *strm->next_out++ = (unsigned char)data; -} - -FLUSH(msb_32) -FLUSH(msb_24) -FLUSH(msb_16) -FLUSH(lsb_32) -FLUSH(lsb_24) -FLUSH(lsb_16) -FLUSH(8) - -static inline void put_sample(struct aec_stream *strm, uint32_t s) -{ - struct internal_state *state = strm->state; - - *state->rsip++ = s; - strm->avail_out -= state->bytes_per_sample; -} - -static inline uint32_t direct_get(struct aec_stream *strm, int n) -{ - /** - Get n bit from input stream - - No checking whatsoever. Read bits are dumped. - */ - - struct internal_state *state = strm->state; - if (state->bitp < n) - { - int b = (63 - state->bitp) >> 3; - if (b == 6) { - state->acc = (state->acc << 48) - | ((uint64_t)strm->next_in[0] << 40) - | ((uint64_t)strm->next_in[1] << 32) - | ((uint64_t)strm->next_in[2] << 24) - | ((uint64_t)strm->next_in[3] << 16) - | ((uint64_t)strm->next_in[4] << 8) - | (uint64_t)strm->next_in[5]; - } else if (b == 7) { - state->acc = (state->acc << 56) - | ((uint64_t)strm->next_in[0] << 48) - | ((uint64_t)strm->next_in[1] << 40) - | ((uint64_t)strm->next_in[2] << 32) - | ((uint64_t)strm->next_in[3] << 24) - | ((uint64_t)strm->next_in[4] << 16) - | ((uint64_t)strm->next_in[5] << 8) - | (uint64_t)strm->next_in[6]; - } else if (b == 5) { - state->acc = (state->acc << 40) - | ((uint64_t)strm->next_in[0] << 32) - | ((uint64_t)strm->next_in[1] << 24) - | ((uint64_t)strm->next_in[2] << 16) - | ((uint64_t)strm->next_in[3] << 8) - | (uint64_t)strm->next_in[4]; - } else if (b == 4) { - state->acc = (state->acc << 32) - | ((uint64_t)strm->next_in[0] << 24) - | ((uint64_t)strm->next_in[1] << 16) - | ((uint64_t)strm->next_in[2] << 8) - | (uint64_t)strm->next_in[3]; - } else if (b == 3) { - state->acc = (state->acc << 24) - | ((uint64_t)strm->next_in[0] << 16) - | ((uint64_t)strm->next_in[1] << 8) - | (uint64_t)strm->next_in[2]; - } else if (b == 2) { - state->acc = (state->acc << 16) - | ((uint64_t)strm->next_in[0] << 8) - | (uint64_t)strm->next_in[1]; - } else if (b == 1) { - state->acc = (state->acc << 8) - | (uint64_t)strm->next_in[0]; - } - strm->next_in += b; - strm->avail_in -= b; - state->bitp += b << 3; - } - - state->bitp -= n; - return (state->acc >> state->bitp) & (UINT64_MAX >> (64 - n)); -} - -static inline uint32_t direct_get_fs(struct aec_stream *strm) -{ - /** - Interpret a Fundamental Sequence from the input buffer. - - Essentially counts the number of 0 bits until a 1 is - encountered. - */ - - uint32_t fs = 0; - struct internal_state *state = strm->state; - - if (state->bitp) - state->acc &= UINT64_MAX >> (64 - state->bitp); - else - state->acc = 0; - - while (state->acc == 0) { - if (strm->avail_in < 7) - return 0; - - state->acc = (state->acc << 56) - | ((uint64_t)strm->next_in[0] << 48) - | ((uint64_t)strm->next_in[1] << 40) - | ((uint64_t)strm->next_in[2] << 32) - | ((uint64_t)strm->next_in[3] << 24) - | ((uint64_t)strm->next_in[4] << 16) - | ((uint64_t)strm->next_in[5] << 8) - | (uint64_t)strm->next_in[6]; - strm->next_in += 7; - strm->avail_in -= 7; - fs += state->bitp; - state->bitp = 56; - } - - { -#ifndef __has_builtin -#define __has_builtin(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if HAVE_DECL___BUILTIN_CLZLL || __has_builtin(__builtin_clzll) - int i = 63 - __builtin_clzll(state->acc); -#elif HAVE_BSR64 - unsigned long i; - _BitScanReverse64(&i, state->acc); -#else - int i = state->bitp - 1; - while ((state->acc & (UINT64_C(1) << i)) == 0) - i--; -#endif - fs += state->bitp - i - 1; - state->bitp = i; - } - return fs; -} - -static inline uint32_t bits_ask(struct aec_stream *strm, int n) -{ - while (strm->state->bitp < n) { - if (strm->avail_in == 0) - return 0; - strm->avail_in--; - strm->state->acc <<= 8; - strm->state->acc |= *strm->next_in++; - strm->state->bitp += 8; - } - return 1; -} - -static inline uint32_t bits_get(struct aec_stream *strm, int n) -{ - return (strm->state->acc >> (strm->state->bitp - n)) - & (UINT64_MAX >> (64 - n)); -} - -static inline void bits_drop(struct aec_stream *strm, int n) -{ - strm->state->bitp -= n; -} - -static inline uint32_t fs_ask(struct aec_stream *strm) -{ - if (bits_ask(strm, 1) == 0) - return 0; - while ((strm->state->acc & (UINT64_C(1) << (strm->state->bitp - 1))) == 0) { - if (strm->state->bitp == 1) { - if (strm->avail_in == 0) - return 0; - strm->avail_in--; - strm->state->acc <<= 8; - strm->state->acc |= *strm->next_in++; - strm->state->bitp += 8; - } - strm->state->fs++; - strm->state->bitp--; - } - return 1; -} - -static inline void fs_drop(struct aec_stream *strm) -{ - strm->state->fs = 0; - strm->state->bitp--; -} - -static inline uint32_t copysample(struct aec_stream *strm) -{ - if (bits_ask(strm, strm->bits_per_sample) == 0 - || strm->avail_out < strm->state->bytes_per_sample) - return 0; - - put_sample(strm, bits_get(strm, strm->bits_per_sample)); - bits_drop(strm, strm->bits_per_sample); - return 1; -} - -static inline int m_id(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - if (strm->avail_in >= strm->state->in_blklen) { - state->id = direct_get(strm, state->id_len); - } else { - if (bits_ask(strm, state->id_len) == 0) { - state->mode = m_id; - return M_EXIT; - } - state->id = bits_get(strm, state->id_len); - bits_drop(strm, state->id_len); - } - state->mode = state->id_table[state->id]; - return(state->mode(strm)); -} - -static int m_next_cds(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - if (state->rsi_size == RSI_USED_SIZE(state)) { - state->flush_output(strm); - state->flush_start = state->rsi_buffer; - state->rsip = state->rsi_buffer; - if (state->pp) { - state->ref = 1; - state->encoded_block_size = strm->block_size - 1; - } - if (strm->flags & AEC_PAD_RSI) - state->bitp -= state->bitp % 8; - } else { - state->ref = 0; - state->encoded_block_size = strm->block_size; - } - return m_id(strm); -} - -static int m_split_output(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - int k = state->id - 1; - - do { - if (bits_ask(strm, k) == 0 || strm->avail_out < state->bytes_per_sample) - return M_EXIT; - if (k) - *state->rsip++ += bits_get(strm, k); - else - state->rsip++; - strm->avail_out -= state->bytes_per_sample; - bits_drop(strm, k); - } while(++state->sample_counter < state->encoded_block_size); - - state->mode = m_next_cds; - return M_CONTINUE; -} - -static int m_split_fs(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - int k = state->id - 1; - - do { - if (fs_ask(strm) == 0) - return M_EXIT; - state->rsip[state->sample_counter] = state->fs << k; - fs_drop(strm); - } while(++state->sample_counter < state->encoded_block_size); - - state->sample_counter = 0; - state->mode = m_split_output; - - return M_CONTINUE; -} - -static int m_split(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (BUFFERSPACE(strm)) { - int k = state->id - 1; - size_t binary_part = (k * state->encoded_block_size) / 8 + 9; - - if (state->ref) - *state->rsip++ = direct_get(strm, strm->bits_per_sample); - - for (size_t i = 0; i < state->encoded_block_size; i++) - state->rsip[i] = direct_get_fs(strm) << k; - - if (k) { - if (strm->avail_in < binary_part) - return M_ERROR; - - for (size_t i = 0; i < state->encoded_block_size; i++) - *state->rsip++ += direct_get(strm, k); - } else { - state->rsip += state->encoded_block_size; - } - - strm->avail_out -= state->out_blklen; - state->mode = m_next_cds; - } else { - if (state->ref && (copysample(strm) == 0)) - return M_EXIT; - state->sample_counter = 0; - state->mode = m_split_fs; - } - return M_CONTINUE; -} - -static int m_zero_output(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - do { - if (strm->avail_out < state->bytes_per_sample) - return M_EXIT; - put_sample(strm, 0); - } while(--state->sample_counter); - - state->mode = m_next_cds; - return M_CONTINUE; -} - -static int m_zero_block(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - uint32_t zero_blocks; - uint32_t zero_samples; - uint32_t zero_bytes; - - if (fs_ask(strm) == 0) - return M_EXIT; - - zero_blocks = state->fs + 1; - fs_drop(strm); - - if (zero_blocks == ROS) { - int b = (int)RSI_USED_SIZE(state) / strm->block_size; - zero_blocks = MIN((int)(strm->rsi - b), 64 - (b % 64)); - } else if (zero_blocks > ROS) { - zero_blocks--; - } - - zero_samples = zero_blocks * strm->block_size - state->ref; - if (state->rsi_size - RSI_USED_SIZE(state) < zero_samples) - return M_ERROR; - - zero_bytes = zero_samples * state->bytes_per_sample; - if (strm->avail_out >= zero_bytes) { - memset(state->rsip, 0, zero_samples * sizeof(uint32_t)); - state->rsip += zero_samples; - strm->avail_out -= zero_bytes; - state->mode = m_next_cds; - } else { - state->sample_counter = zero_samples; - state->mode = m_zero_output; - } - return M_CONTINUE; -} - -static int m_se_decode(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - while(state->sample_counter < strm->block_size) { - int32_t m; - int32_t d1; - if (fs_ask(strm) == 0) - return M_EXIT; - m = state->fs; - if (m > SE_TABLE_SIZE) - return M_ERROR; - d1 = m - state->se_table[2 * m + 1]; - - if ((state->sample_counter & 1) == 0) { - if (strm->avail_out < state->bytes_per_sample) - return M_EXIT; - put_sample(strm, state->se_table[2 * m] - d1); - state->sample_counter++; - } - - if (strm->avail_out < state->bytes_per_sample) - return M_EXIT; - put_sample(strm, d1); - state->sample_counter++; - fs_drop(strm); - } - - state->mode = m_next_cds; - return M_CONTINUE; -} - -static int m_se(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (BUFFERSPACE(strm)) { - uint32_t i = state->ref; - - while (i < strm->block_size) { - uint32_t m = direct_get_fs(strm); - int32_t d1; - - if (m > SE_TABLE_SIZE) - return M_ERROR; - - d1 = m - state->se_table[2 * m + 1]; - - if ((i & 1) == 0) { - put_sample(strm, state->se_table[2 * m] - d1); - i++; - } - put_sample(strm, d1); - i++; - } - state->mode = m_next_cds; - } else { - state->mode = m_se_decode; - state->sample_counter = state->ref; - } - return M_CONTINUE; -} - -static int m_low_entropy_ref(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (state->ref && copysample(strm) == 0) - return M_EXIT; - - if (state->id == 1) - state->mode = m_se; - else - state->mode = m_zero_block; - return M_CONTINUE; -} - -static int m_low_entropy(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (bits_ask(strm, 1) == 0) - return M_EXIT; - state->id = bits_get(strm, 1); - bits_drop(strm, 1); - state->mode = m_low_entropy_ref; - return M_CONTINUE; -} - -static int m_uncomp_copy(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - do { - if (copysample(strm) == 0) - return M_EXIT; - } while(--state->sample_counter); - - state->mode = m_next_cds; - return M_CONTINUE; -} - -static int m_uncomp(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (BUFFERSPACE(strm)) { - for (size_t i = 0; i < strm->block_size; i++) - *state->rsip++ = direct_get(strm, strm->bits_per_sample); - strm->avail_out -= state->out_blklen; - state->mode = m_next_cds; - } else { - state->sample_counter = strm->block_size; - state->mode = m_uncomp_copy; - } - return M_CONTINUE; -} - -static void create_se_table(int *table) -{ - int k = 0; - for (int i = 0; i < 13; i++) { - int ms = k; - for (int j = 0; j <= i; j++) { - table[2 * k] = i; - table[2 * k + 1] = ms; - k++; - } - } -} - -int aec_decode_init(struct aec_stream *strm) -{ - struct internal_state *state; - int modi; - - if (strm->bits_per_sample > 32 || strm->bits_per_sample == 0) - return AEC_CONF_ERROR; - - state = malloc(sizeof(struct internal_state)); - if (state == NULL) - return AEC_MEM_ERROR; - memset(state, 0, sizeof(struct internal_state)); - - create_se_table(state->se_table); - - strm->state = state; - - if (strm->bits_per_sample > 16) { - state->id_len = 5; - - if (strm->bits_per_sample <= 24 && strm->flags & AEC_DATA_3BYTE) { - state->bytes_per_sample = 3; - if (strm->flags & AEC_DATA_MSB) - state->flush_output = flush_msb_24; - else - state->flush_output = flush_lsb_24; - } else { - state->bytes_per_sample = 4; - if (strm->flags & AEC_DATA_MSB) - state->flush_output = flush_msb_32; - else - state->flush_output = flush_lsb_32; - } - state->out_blklen = strm->block_size * state->bytes_per_sample; - } - else if (strm->bits_per_sample > 8) { - state->bytes_per_sample = 2; - state->id_len = 4; - state->out_blklen = strm->block_size * 2; - if (strm->flags & AEC_DATA_MSB) - state->flush_output = flush_msb_16; - else - state->flush_output = flush_lsb_16; - } else { - if (strm->flags & AEC_RESTRICTED) { - if (strm->bits_per_sample <= 4) { - if (strm->bits_per_sample <= 2) - state->id_len = 1; - else - state->id_len = 2; - } else { - return AEC_CONF_ERROR; - } - } else { - state->id_len = 3; - } - - state->bytes_per_sample = 1; - state->out_blklen = strm->block_size; - state->flush_output = flush_8; - } - - if (strm->flags & AEC_DATA_SIGNED) { - state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); - state->xmin = ~state->xmax; - } else { - state->xmin = 0; - state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample); - } - - state->in_blklen = (strm->block_size * strm->bits_per_sample - + state->id_len) / 8 + 16; - - modi = 1UL << state->id_len; - state->id_table = malloc(modi * sizeof(int (*)(struct aec_stream *))); - if (state->id_table == NULL) - return AEC_MEM_ERROR; - - state->id_table[0] = m_low_entropy; - for (int i = 1; i < modi - 1; i++) { - state->id_table[i] = m_split; - } - state->id_table[modi - 1] = m_uncomp; - - state->rsi_size = strm->rsi * strm->block_size; - state->rsi_buffer = malloc(state->rsi_size * sizeof(uint32_t)); - if (state->rsi_buffer == NULL) - return AEC_MEM_ERROR; - - state->pp = strm->flags & AEC_DATA_PREPROCESS; - if (state->pp) { - state->ref = 1; - state->encoded_block_size = strm->block_size - 1; - } else { - state->ref = 0; - state->encoded_block_size = strm->block_size; - } - strm->total_in = 0; - strm->total_out = 0; - - state->rsip = state->rsi_buffer; - state->flush_start = state->rsi_buffer; - state->bitp = 0; - state->fs = 0; - state->mode = m_id; - return AEC_OK; -} - -int aec_decode(struct aec_stream *strm, int flush) -{ - /** - Finite-state machine implementation of the adaptive entropy - decoder. - - Can work with one byte input und one sample output buffers. If - enough buffer space is available, then faster implementations - of the states are called. Inspired by zlib. - */ - - struct internal_state *state = strm->state; - int status; - - strm->total_in += strm->avail_in; - strm->total_out += strm->avail_out; - - do { - status = state->mode(strm); - } while (status == M_CONTINUE); - - if (status == M_ERROR) - return AEC_DATA_ERROR; - - if (status == M_EXIT && strm->avail_out > 0 && - strm->avail_out < state->bytes_per_sample) - return AEC_MEM_ERROR; - - state->flush_output(strm); - - strm->total_in -= strm->avail_in; - strm->total_out -= strm->avail_out; - - return AEC_OK; -} - -int aec_decode_end(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - free(state->id_table); - free(state->rsi_buffer); - free(state); - return AEC_OK; -} - -int aec_buffer_decode(struct aec_stream *strm) -{ - int status = aec_decode_init(strm); - if (status != AEC_OK) - return status; - - status = aec_decode(strm, AEC_FLUSH); - aec_decode_end(strm); - return status; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.h deleted file mode 100644 index 78cdbcb..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/decode.h +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @file decode.c - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Adaptive Entropy Decoder - * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 - * - */ - -#ifndef DECODE_H -#define DECODE_H 1 - -#include - -#if HAVE_STDINT_H -# include -#endif - -#define M_CONTINUE 1 -#define M_EXIT 0 -#define M_ERROR (-1) - -#define MIN(a, b) (((a) < (b))? (a): (b)) - -#define SE_TABLE_SIZE 90 - -struct aec_stream; - -struct internal_state { - int (*mode)(struct aec_stream *); - - /* option ID */ - int id; - - /* bit length of code option identification key */ - int id_len; - - /* table maps IDs to states */ - int (**id_table)(struct aec_stream *); - - void (*flush_output)(struct aec_stream *); - - /* previous output for post-processing */ - int32_t last_out; - - /* minimum integer for post-processing */ - uint32_t xmin; - - /* maximum integer for post-processing */ - uint32_t xmax; - - /* length of uncompressed input block should be the longest - legal block */ - uint32_t in_blklen; - - /* length of output block in bytes */ - uint32_t out_blklen; - - uint32_t sample_counter; - - /* accumulator for currently used bit sequence */ - uint64_t acc; - - /* bit pointer to the next unused bit in accumulator */ - int bitp; - - /* last fundamental sequence in accumulator */ - int fs; - - /* 1 if current block has reference sample */ - int ref; - - /* block_size minus reference sample if present */ - uint32_t encoded_block_size; - - /* 1 if postprocessor has to be used */ - int pp; - - /* storage size of samples in bytes */ - uint32_t bytes_per_sample; - - /* output buffer holding one reference sample interval */ - uint32_t *rsi_buffer; - - /* current position of output in rsi_buffer */ - uint32_t *rsip; - - /* rsi in bytes */ - size_t rsi_size; - - /* first not yet flushed byte in rsi_buffer */ - uint32_t *flush_start; - - /* table for decoding second extension option */ - int se_table[2 * (SE_TABLE_SIZE + 1)]; -} decode_state; - -#endif /* DECODE_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.c deleted file mode 100644 index cbaf8af..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.c +++ /dev/null @@ -1,948 +0,0 @@ -/** - * @file encode.c - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Adaptive Entropy Encoder - * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 - * - */ - -#include -#include -#include - -#include "encode.h" -#include "encode_accessors.h" -#include "libaec.h" - -static int m_get_block(struct aec_stream *strm); - -static inline void emit(struct internal_state *state, - uint32_t data, int bits) -{ - /** - Emit sequence of bits. - */ - - if (bits <= state->bits) { - state->bits -= bits; - *state->cds += (uint8_t)(data << state->bits); - } else { - bits -= state->bits; - *state->cds++ += (uint8_t)((uint64_t)data >> bits); - - while (bits > 8) { - bits -= 8; - *state->cds++ = (uint8_t)(data >> bits); - } - - state->bits = 8 - bits; - *state->cds = (uint8_t)(data << state->bits); - } -} - -static inline void emitfs(struct internal_state *state, int fs) -{ - /** - Emits a fundamental sequence. - - fs zero bits followed by one 1 bit. - */ - - for(;;) { - if (fs < state->bits) { - state->bits -= fs + 1; - *state->cds += 1U << state->bits; - break; - } else { - fs -= state->bits; - *++state->cds = 0; - state->bits = 8; - } - } -} - -static inline void copy64(uint8_t *dst, uint64_t src) -{ - dst[0] = (uint8_t)(src >> 56); - dst[1] = (uint8_t)(src >> 48); - dst[2] = (uint8_t)(src >> 40); - dst[3] = (uint8_t)(src >> 32); - dst[4] = (uint8_t)(src >> 24); - dst[5] = (uint8_t)(src >> 16); - dst[6] = (uint8_t)(src >> 8); - dst[7] = (uint8_t)src; -} - -static inline void emitblock_fs(struct aec_stream *strm, int k, int ref) -{ - struct internal_state *state = strm->state; - - uint64_t acc = (uint64_t)*state->cds << 56; - uint32_t used = 7 - state->bits; /* used bits in 64 bit accumulator */ - - for (size_t i = ref; i < strm->block_size; i++) { - used += (state->block[i] >> k) + 1; - while (used > 63) { - copy64(state->cds, acc); - state->cds += 8; - acc = 0; - used -= 64; - } - acc |= UINT64_C(1) << (63 - used); - } - - copy64(state->cds, acc); - state->cds += used >> 3; - state->bits = 7 - (used & 7); -} - -static inline void emitblock(struct aec_stream *strm, int k, int ref) -{ - /** - Emit the k LSB of a whole block of input data. - */ - - struct internal_state *state = strm->state; - uint32_t *in = state->block + ref; - uint32_t *in_end = state->block + strm->block_size; - uint64_t mask = (UINT64_C(1) << k) - 1; - uint8_t *o = state->cds; - uint64_t a = *o; - int p = state->bits; - - while(in < in_end) { - a <<= 56; - p = (p % 8) + 56; - - while (p > k && in < in_end) { - p -= k; - a += ((uint64_t)(*in++) & mask) << p; - } - - switch (p & ~7) { - case 0: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - o[2] = (uint8_t)(a >> 40); - o[3] = (uint8_t)(a >> 32); - o[4] = (uint8_t)(a >> 24); - o[5] = (uint8_t)(a >> 16); - o[6] = (uint8_t)(a >> 8); - o += 7; - break; - case 8: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - o[2] = (uint8_t)(a >> 40); - o[3] = (uint8_t)(a >> 32); - o[4] = (uint8_t)(a >> 24); - o[5] = (uint8_t)(a >> 16); - a >>= 8; - o += 6; - break; - case 16: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - o[2] = (uint8_t)(a >> 40); - o[3] = (uint8_t)(a >> 32); - o[4] = (uint8_t)(a >> 24); - a >>= 16; - o += 5; - break; - case 24: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - o[2] = (uint8_t)(a >> 40); - o[3] = (uint8_t)(a >> 32); - a >>= 24; - o += 4; - break; - case 32: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - o[2] = (uint8_t)(a >> 40); - a >>= 32; - o += 3; - break; - case 40: - o[0] = (uint8_t)(a >> 56); - o[1] = (uint8_t)(a >> 48); - a >>= 40; - o += 2; - break; - case 48: - *o++ = (uint8_t)(a >> 56); - a >>= 48; - break; - default: - a >>= 56; - break; - } - } - - *o = (uint8_t)a; - state->cds = o; - state->bits = p % 8; -} - -static void preprocess_unsigned(struct aec_stream *strm) -{ - /** - Preprocess RSI of unsigned samples. - - Combining preprocessing and converting to uint32_t in one loop - is slower due to the data dependance on x_i-1. - */ - - uint32_t D; - struct internal_state *state = strm->state; - const uint32_t *restrict x = state->data_raw; - uint32_t *restrict d = state->data_pp; - uint32_t xmax = state->xmax; - uint32_t rsi = strm->rsi * strm->block_size - 1; - - state->ref = 1; - state->ref_sample = x[0]; - d[0] = 0; - for (size_t i = 0; i < rsi; i++) { - if (x[i + 1] >= x[i]) { - D = x[i + 1] - x[i]; - if (D <= x[i]) - d[i + 1] = 2 * D; - else - d[i + 1] = x[i + 1]; - } else { - D = x[i] - x[i + 1]; - if (D <= xmax - x[i]) - d[i + 1] = 2 * D - 1; - else - d[i + 1] = xmax - x[i + 1]; - } - } - state->uncomp_len = (strm->block_size - 1) * strm->bits_per_sample; -} - -static void preprocess_signed(struct aec_stream *strm) -{ - /** - Preprocess RSI of signed samples. - */ - - uint32_t D; - struct internal_state *state = strm->state; - int32_t *restrict x = (int32_t *)state->data_raw; - uint32_t *restrict d = state->data_pp; - int32_t xmax = (int32_t)state->xmax; - int32_t xmin = (int32_t)state->xmin; - uint32_t rsi = strm->rsi * strm->block_size - 1; - uint32_t m = UINT64_C(1) << (strm->bits_per_sample - 1); - - state->ref = 1; - state->ref_sample = x[0]; - d[0] = 0; - x[0] = (x[0] ^ m) - m; - - for (size_t i = 0; i < rsi; i++) { - x[i + 1] = (x[i + 1] ^ m) - m; - if (x[i + 1] < x[i]) { - D = (uint32_t)(x[i] - x[i + 1]); - if (D <= (uint32_t)(xmax - x[i])) - d[i + 1] = 2 * D - 1; - else - d[i + 1] = xmax - x[i + 1]; - } else { - D = (uint32_t)(x[i + 1] - x[i]); - if (D <= (uint32_t)(x[i] - xmin)) - d[i + 1] = 2 * D; - else - d[i + 1] = x[i + 1] - xmin; - } - } - state->uncomp_len = (strm->block_size - 1) * strm->bits_per_sample; -} - -static inline uint64_t block_fs(struct aec_stream *strm, int k) -{ - /** - Sum FS of all samples in block for given splitting position. - */ - - struct internal_state *state = strm->state; - uint64_t fs = 0; - - for (size_t i = 0; i < strm->block_size; i++) - fs += (uint64_t)(state->block[i] >> k); - - return fs; -} - -static uint32_t assess_splitting_option(struct aec_stream *strm) -{ - /** - Length of CDS encoded with splitting option and optimal k. - - In Rice coding each sample in a block of samples is split at - the same position into k LSB and bits_per_sample - k MSB. The - LSB part is left binary and the MSB part is coded as a - fundamental sequence a.k.a. unary (see CCSDS 121.0-B-2). The - function of the length of the Coded Data Set (CDS) depending on - k has exactly one minimum (see A. Kiely, IPN Progress Report - 42-159). - - To find that minimum with only a few costly evaluations of the - CDS length, we start with the k of the previous CDS. K is - increased and the CDS length evaluated. If the CDS length gets - smaller, then we are moving towards the minimum. If the length - increases, then the minimum will be found with smaller k. - - For increasing k we know that we will gain block_size bits in - length through the larger binary part. If the FS lenth is less - than the block size then a reduced FS part can't compensate the - larger binary part. So we know that the CDS for k+1 will be - larger than for k without actually computing the length. An - analogue check can be done for decreasing k. - */ - - struct internal_state *state = strm->state; - - /* Block size of current block */ - int this_bs = strm->block_size - state->ref; - - /* CDS length minimum so far */ - uint64_t len_min = UINT64_MAX; - - int k = state->k; - int k_min = k; - - /* 1 if we shouldn't reverse */ - int no_turn = (k == 0); - - /* Direction, 1 means increasing k, 0 decreasing k */ - int dir = 1; - - for (;;) { - /* Length of FS part (not including 1s) */ - uint64_t fs_len = block_fs(strm, k); - - /* CDS length for current k */ - uint64_t len = fs_len + this_bs * (k + 1); - - if (len < len_min) { - if (len_min < UINT64_MAX) - no_turn = 1; - - len_min = len; - k_min = k; - - if (dir) { - if (fs_len < this_bs || k >= state->kmax) { - if (no_turn) - break; - k = state->k - 1; - dir = 0; - no_turn = 1; - } else { - k++; - } - } else { - if (fs_len >= this_bs || k == 0) - break; - k--; - } - } else { - if (no_turn) - break; - k = state->k - 1; - dir = 0; - no_turn = 1; - } - } - state->k = k_min; - - return (uint32_t)len_min; -} - -static uint32_t assess_se_option(struct aec_stream *strm) -{ - /** - Length of CDS encoded with Second Extension option. - - If length is above limit just return UINT32_MAX. - */ - - struct internal_state *state = strm->state; - uint32_t *block = state->block; - uint64_t len = 1; - - for (size_t i = 0; i < strm->block_size; i += 2) { - uint64_t d = (uint64_t)block[i] + (uint64_t)block[i + 1]; - len += d * (d + 1) / 2 + block[i + 1] + 1; - if (len > state->uncomp_len) - return UINT32_MAX; - } - return (uint32_t)len; -} - -static void init_output(struct aec_stream *strm) -{ - /** - Direct output to next_out if next_out can hold a Coded Data - Set, use internal buffer otherwise. - */ - - struct internal_state *state = strm->state; - - if (strm->avail_out > CDSLEN) { - if (!state->direct_out) { - state->direct_out = 1; - *strm->next_out = *state->cds; - state->cds = strm->next_out; - } - } else { - if (state->zero_blocks == 0 || state->direct_out) { - /* copy leftover from last block */ - *state->cds_buf = *state->cds; - state->cds = state->cds_buf; - } - state->direct_out = 0; - } -} - -/* - * - * FSM functions - * - */ - -static int m_flush_block_resumable(struct aec_stream *strm) -{ - /** - Slow and restartable flushing - */ - struct internal_state *state = strm->state; - - int n = (int)MIN((size_t)(state->cds - state->cds_buf - state->i), - strm->avail_out); - memcpy(strm->next_out, state->cds_buf + state->i, n); - strm->next_out += n; - strm->avail_out -= n; - state->i += n; - - if (strm->avail_out == 0) { - return M_EXIT; - } else { - state->mode = m_get_block; - return M_CONTINUE; - } -} - -static int m_flush_block(struct aec_stream *strm) -{ - /** - Flush block in direct_out mode by updating counters. - - Fall back to slow flushing if in buffered mode. - */ - struct internal_state *state = strm->state; - -#ifdef ENABLE_RSI_PADDING - if (state->blocks_avail == 0 - && strm->flags & AEC_PAD_RSI - && state->block_nonzero == 0 - ) - emit(state, 0, state->bits % 8); -#endif - - if (state->direct_out) { - int n = (int)(state->cds - strm->next_out); - strm->next_out += n; - strm->avail_out -= n; - state->mode = m_get_block; - return M_CONTINUE; - } - - state->i = 0; - state->mode = m_flush_block_resumable; - return M_CONTINUE; -} - -static int m_encode_splitting(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - int k = state->k; - - emit(state, k + 1, state->id_len); - if (state->ref) - emit(state, state->ref_sample, strm->bits_per_sample); - - emitblock_fs(strm, k, state->ref); - if (k) - emitblock(strm, k, state->ref); - - return m_flush_block(strm); -} - -static int m_encode_uncomp(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - emit(state, (1U << state->id_len) - 1, state->id_len); - if (state->ref) - state->block[0] = state->ref_sample; - emitblock(strm, strm->bits_per_sample, 0); - return m_flush_block(strm); -} - -static int m_encode_se(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - emit(state, 1, state->id_len + 1); - if (state->ref) - emit(state, state->ref_sample, strm->bits_per_sample); - - for (size_t i = 0; i < strm->block_size; i+= 2) { - uint32_t d = state->block[i] + state->block[i + 1]; - emitfs(state, d * (d + 1) / 2 + state->block[i + 1]); - } - - return m_flush_block(strm); -} - -static int m_encode_zero(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - emit(state, 0, state->id_len + 1); - - if (state->zero_ref) - emit(state, state->zero_ref_sample, strm->bits_per_sample); - - if (state->zero_blocks == ROS) - emitfs(state, 4); - else if (state->zero_blocks >= 5) - emitfs(state, state->zero_blocks); - else - emitfs(state, state->zero_blocks - 1); - - state->zero_blocks = 0; - return m_flush_block(strm); -} - -static int m_select_code_option(struct aec_stream *strm) -{ - /** - Decide which code option to use. - */ - - struct internal_state *state = strm->state; - - uint32_t split_len; - uint32_t se_len; - if (state->id_len > 1) - split_len = assess_splitting_option(strm); - else - split_len = UINT32_MAX; - se_len = assess_se_option(strm); - - if (split_len < state->uncomp_len) { - if (split_len < se_len) - return m_encode_splitting(strm); - else - return m_encode_se(strm); - } else { - if (state->uncomp_len <= se_len) - return m_encode_uncomp(strm); - else - return m_encode_se(strm); - } -} - -static int m_check_zero_block(struct aec_stream *strm) -{ - /** - Check if input block is all zero. - - Aggregate consecutive zero blocks until we find !0 or reach the - end of a segment or RSI. - */ - - struct internal_state *state = strm->state; - uint32_t *p = state->block; - - size_t i; - for (i = 0; i < strm->block_size; i++) - if (p[i] != 0) - break; - - if (i < strm->block_size) { - if (state->zero_blocks) { - /* The current block isn't zero but we have to emit a - * previous zero block first. The current block will be - * flagged and handled later. - */ - state->block_nonzero = 1; - state->mode = m_encode_zero; - return M_CONTINUE; - } - state->mode = m_select_code_option; - return M_CONTINUE; - } else { - state->zero_blocks++; - if (state->zero_blocks == 1) { - state->zero_ref = state->ref; - state->zero_ref_sample = state->ref_sample; - } - if (state->blocks_avail == 0 || state->blocks_dispensed % 64 == 0) { - if (state->zero_blocks > 4) - state->zero_blocks = ROS; - - state->mode = m_encode_zero; - return M_CONTINUE; - } - state->mode = m_get_block; - return M_CONTINUE; - } -} - -static int m_get_rsi_resumable(struct aec_stream *strm) -{ - /** - Get RSI while input buffer is short. - - Let user provide more input. Once we got all input pad buffer - to full RSI. - */ - - struct internal_state *state = strm->state; - - do { - if (strm->avail_in >= state->bytes_per_sample) { - state->data_raw[state->i] = state->get_sample(strm); - } else { - if (state->flush == AEC_FLUSH) { - if (state->i > 0) { - state->blocks_avail = state->i / strm->block_size - 1; - if (state->i % strm->block_size) - state->blocks_avail++; - do - state->data_raw[state->i] = - state->data_raw[state->i - 1]; - while(++state->i < strm->rsi * strm->block_size); - } else { - /* Finish encoding by padding the last byte with - * zero bits. */ - emit(state, 0, state->bits); - if (strm->avail_out > 0) { - if (!state->direct_out) - *strm->next_out++ = *state->cds; - strm->avail_out--; - state->flushed = 1; - } - return M_EXIT; - } - } else { - return M_EXIT; - } - } - } while (++state->i < strm->rsi * strm->block_size); - - if (strm->flags & AEC_DATA_PREPROCESS) - state->preprocess(strm); - - return m_check_zero_block(strm); -} - -static int m_get_block(struct aec_stream *strm) -{ - /** - Provide the next block of preprocessed input data. - - Pull in a whole Reference Sample Interval (RSI) of data if - block buffer is empty. - */ - - struct internal_state *state = strm->state; - - init_output(strm); - - if (state->block_nonzero) { - state->block_nonzero = 0; - state->mode = m_select_code_option; - return M_CONTINUE; - } - - if (state->blocks_avail == 0) { - state->blocks_avail = strm->rsi - 1; - state->block = state->data_pp; - state->blocks_dispensed = 1; - - if (strm->avail_in >= state->rsi_len) { - state->get_rsi(strm); - if (strm->flags & AEC_DATA_PREPROCESS) - state->preprocess(strm); - - return m_check_zero_block(strm); - } else { - state->i = 0; - state->mode = m_get_rsi_resumable; - } - } else { - if (state->ref) { - state->ref = 0; - state->uncomp_len = strm->block_size * strm->bits_per_sample; - } - state->block += strm->block_size; - state->blocks_dispensed++; - state->blocks_avail--; - return m_check_zero_block(strm); - } - return M_CONTINUE; -} - -static void cleanup(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - if (strm->flags & AEC_DATA_PREPROCESS && state->data_raw) - free(state->data_raw); - if (state->data_pp) - free(state->data_pp); - free(state); -} - -/* - * - * API functions - * - */ - -int aec_encode_init(struct aec_stream *strm) -{ - struct internal_state *state; - - if (strm->bits_per_sample > 32 || strm->bits_per_sample == 0) - return AEC_CONF_ERROR; - - if (strm->flags & AEC_NOT_ENFORCE) { - /* All even block sizes are allowed. */ - if (strm->block_size & 1) - return AEC_CONF_ERROR; - } else { - /* Only allow standard conforming block sizes */ - if (strm->block_size != 8 - && strm->block_size != 16 - && strm->block_size != 32 - && strm->block_size != 64) - return AEC_CONF_ERROR; - } - - if (strm->rsi > 4096) - return AEC_CONF_ERROR; - - state = malloc(sizeof(struct internal_state)); - if (state == NULL) - return AEC_MEM_ERROR; - - memset(state, 0, sizeof(struct internal_state)); - strm->state = state; - state->uncomp_len = strm->block_size * strm->bits_per_sample; - - if (strm->bits_per_sample > 16) { - /* 24/32 input bit settings */ - state->id_len = 5; - - if (strm->bits_per_sample <= 24 - && strm->flags & AEC_DATA_3BYTE) { - state->bytes_per_sample = 3; - if (strm->flags & AEC_DATA_MSB) { - state->get_sample = aec_get_msb_24; - state->get_rsi = aec_get_rsi_msb_24; - } else { - state->get_sample = aec_get_lsb_24; - state->get_rsi = aec_get_rsi_lsb_24; - } - } else { - state->bytes_per_sample = 4; - if (strm->flags & AEC_DATA_MSB) { - state->get_sample = aec_get_msb_32; - state->get_rsi = aec_get_rsi_msb_32; - } else { - state->get_sample = aec_get_lsb_32; - state->get_rsi = aec_get_rsi_lsb_32; - } - } - } - else if (strm->bits_per_sample > 8) { - /* 16 bit settings */ - state->id_len = 4; - state->bytes_per_sample = 2; - - if (strm->flags & AEC_DATA_MSB) { - state->get_sample = aec_get_msb_16; - state->get_rsi = aec_get_rsi_msb_16; - } else { - state->get_sample = aec_get_lsb_16; - state->get_rsi = aec_get_rsi_lsb_16; - } - } else { - /* 8 bit settings */ - if (strm->flags & AEC_RESTRICTED) { - if (strm->bits_per_sample <= 4) { - if (strm->bits_per_sample <= 2) - state->id_len = 1; - else - state->id_len = 2; - } else { - return AEC_CONF_ERROR; - } - } else { - state->id_len = 3; - } - state->bytes_per_sample = 1; - - state->get_sample = aec_get_8; - state->get_rsi = aec_get_rsi_8; - } - state->rsi_len = strm->rsi * strm->block_size * state->bytes_per_sample; - - if (strm->flags & AEC_DATA_SIGNED) { - state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); - state->xmin = ~state->xmax; - state->preprocess = preprocess_signed; - } else { - state->xmin = 0; - state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample); - state->preprocess = preprocess_unsigned; - } - - state->kmax = (1U << state->id_len) - 3; - - state->data_pp = malloc(strm->rsi - * strm->block_size - * sizeof(uint32_t)); - if (state->data_pp == NULL) { - cleanup(strm); - return AEC_MEM_ERROR; - } - - if (strm->flags & AEC_DATA_PREPROCESS) { - state->data_raw = malloc(strm->rsi - * strm->block_size - * sizeof(uint32_t)); - if (state->data_raw == NULL) { - cleanup(strm); - return AEC_MEM_ERROR; - } - } else { - state->data_raw = state->data_pp; - } - - state->block = state->data_pp; - - state->ref = 0; - strm->total_in = 0; - strm->total_out = 0; - state->flushed = 0; - - state->cds = state->cds_buf; - *state->cds = 0; - state->bits = 8; - state->mode = m_get_block; - - return AEC_OK; -} - -int aec_encode(struct aec_stream *strm, int flush) -{ - /** - Finite-state machine implementation of the adaptive entropy - encoder. - */ - struct internal_state *state = strm->state; - - state->flush = flush; - strm->total_in += strm->avail_in; - strm->total_out += strm->avail_out; - - while (state->mode(strm) == M_CONTINUE); - - if (state->direct_out) { - int n = (int)(state->cds - strm->next_out); - strm->next_out += n; - strm->avail_out -= n; - - *state->cds_buf = *state->cds; - state->cds = state->cds_buf; - state->direct_out = 0; - } - strm->total_in -= strm->avail_in; - strm->total_out -= strm->avail_out; - return AEC_OK; -} - -int aec_encode_end(struct aec_stream *strm) -{ - struct internal_state *state = strm->state; - - int status = AEC_OK; - if (state->flush == AEC_FLUSH && state->flushed == 0) - status = AEC_STREAM_ERROR; - cleanup(strm); - return status; -} - -int aec_buffer_encode(struct aec_stream *strm) -{ - int status = aec_encode_init(strm); - if (status != AEC_OK) - return status; - status = aec_encode(strm, AEC_FLUSH); - if (status != AEC_OK) { - cleanup(strm); - return status; - } - return aec_encode_end(strm); -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.h deleted file mode 100644 index 9622c44..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode.h +++ /dev/null @@ -1,160 +0,0 @@ -/** - * @file encode.h - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Adaptive Entropy Encoder - * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 - * - */ - -#ifndef ENCODE_H -#define ENCODE_H 1 - -#include - -#if HAVE_STDINT_H -# include -#endif - -#define M_CONTINUE 1 -#define M_EXIT 0 -#define MIN(a, b) (((a) < (b))? (a): (b)) - -/* Maximum CDS length in bytes: 5 bits ID, 64 * 32 bits samples, 7 - * bits carry from previous CDS */ -#define CDSLEN ((5 + 64 * 32 + 7 + 7) / 8) - -/* Marker for Remainder Of Segment condition in zero block encoding */ -#define ROS -1 - -struct aec_stream; - -struct internal_state { - int (*mode)(struct aec_stream *); - uint32_t (*get_sample)(struct aec_stream *); - void (*get_rsi)(struct aec_stream *); - void (*preprocess)(struct aec_stream *); - - /* bit length of code option identification key */ - int id_len; - - /* minimum integer for preprocessing */ - uint32_t xmin; - - /* maximum integer for preprocessing */ - uint32_t xmax; - - uint32_t i; - - /* RSI blocks of preprocessed input */ - uint32_t *data_pp; - - /* RSI blocks of input */ - uint32_t *data_raw; - - /* remaining blocks in buffer */ - int blocks_avail; - - /* blocks encoded so far in RSI */ - int blocks_dispensed; - - /* current (preprocessed) input block */ - uint32_t *block; - - /* reference sample interval in byte */ - uint32_t rsi_len; - - /* current Coded Data Set output */ - uint8_t *cds; - - /* buffer for one CDS (only used if strm->next_out cannot hold - * full CDS) */ - uint8_t cds_buf[CDSLEN]; - - /* cds points to strm->next_out (1) or cds_buf (0) */ - int direct_out; - - /* Free bits (LSB) in output buffer or accumulator */ - int bits; - - /* length of reference sample in current block i.e. 0 or 1 - * depending on whether the block has a reference sample or not */ - int ref; - - /* reference sample stored here for performance reasons */ - uint32_t ref_sample; - - /* current zero block has a reference sample */ - int zero_ref; - - /* reference sample of zero block */ - uint32_t zero_ref_sample; - - /* storage size of samples in bytes */ - uint32_t bytes_per_sample; - - /* number of contiguous zero blocks */ - int zero_blocks; - - /* 1 if this is the first non-zero block after one or more zero - * blocks */ - int block_nonzero; - - /* splitting position */ - int k; - - /* maximum number for k depending on id_len */ - int kmax; - - /* flush option copied from argument */ - int flush; - - /* 1 if flushing was successful */ - int flushed; - - /* length of uncompressed CDS */ - uint32_t uncomp_len; -}; - -#endif /* ENCODE_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.c deleted file mode 100644 index a3ef9b9..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.c +++ /dev/null @@ -1,250 +0,0 @@ -/** - * @file encode_accessors.c - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Read various data types from input stream - * - */ - -#include - -#if HAVE_STDINT_H -# include -#endif - -#include -#include "libaec.h" -#include "encode.h" -#include "encode_accessors.h" - -uint32_t aec_get_8(struct aec_stream *strm) -{ - strm->avail_in--; - return *strm->next_in++; -} - -uint32_t aec_get_lsb_16(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[1] << 8) - | (uint32_t)strm->next_in[0]; - - strm->next_in += 2; - strm->avail_in -= 2; - return data; -} - -uint32_t aec_get_msb_16(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[0] << 8) - | (uint32_t)strm->next_in[1]; - - strm->next_in += 2; - strm->avail_in -= 2; - return data; -} - -uint32_t aec_get_lsb_24(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[2] << 16) - | ((uint32_t)strm->next_in[1] << 8) - | (uint32_t)strm->next_in[0]; - - strm->next_in += 3; - strm->avail_in -= 3; - return data; -} - -uint32_t aec_get_msb_24(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[0] << 16) - | ((uint32_t)strm->next_in[1] << 8) - | (uint32_t)strm->next_in[2]; - - strm->next_in += 3; - strm->avail_in -= 3; - return data; -} - -uint32_t aec_get_lsb_32(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[3] << 24) - | ((uint32_t)strm->next_in[2] << 16) - | ((uint32_t)strm->next_in[1] << 8) - | (uint32_t)strm->next_in[0]; - - strm->next_in += 4; - strm->avail_in -= 4; - return data; -} - -uint32_t aec_get_msb_32(struct aec_stream *strm) -{ - uint32_t data = ((uint32_t)strm->next_in[0] << 24) - | ((uint32_t)strm->next_in[1] << 16) - | ((uint32_t)strm->next_in[2] << 8) - | (uint32_t)strm->next_in[3]; - - strm->next_in += 4; - strm->avail_in -= 4; - return data; -} - -void aec_get_rsi_8(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - unsigned const char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = (uint32_t)in[i]; - - strm->next_in += rsi; - strm->avail_in -= rsi; -} - -void aec_get_rsi_lsb_16(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = (uint32_t)in[2 * i] | ((uint32_t)in[2 * i + 1] << 8); - - strm->next_in += 2 * rsi; - strm->avail_in -= 2 * rsi; -} - -void aec_get_rsi_msb_16(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = ((uint32_t)in[2 * i] << 8) | (uint32_t)in[2 * i + 1]; - - strm->next_in += 2 * rsi; - strm->avail_in -= 2 * rsi; -} - -void aec_get_rsi_lsb_24(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = (uint32_t)in[3 * i] - | ((uint32_t)in[3 * i + 1] << 8) - | ((uint32_t)in[3 * i + 2] << 16); - - strm->next_in += 3 * rsi; - strm->avail_in -= 3 * rsi; -} - -void aec_get_rsi_msb_24(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = ((uint32_t)in[3 * i] << 16) - | ((uint32_t)in[3 * i + 1] << 8) - | (uint32_t)in[3 * i + 2]; - - strm->next_in += 3 * rsi; - strm->avail_in -= 3 * rsi; -} - -#define AEC_GET_RSI_NATIVE_32(BO) \ - void aec_get_rsi_##BO##_32(struct aec_stream *strm) \ - { \ - int rsi = strm->rsi * strm->block_size; \ - memcpy(strm->state->data_raw, \ - strm->next_in, 4 * rsi); \ - strm->next_in += 4 * rsi; \ - strm->avail_in -= 4 * rsi; \ - } - -#ifdef WORDS_BIGENDIAN -void aec_get_rsi_lsb_32(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - for (int i = 0; i < rsi; i++) - out[i] = (uint32_t)in[4 * i] - | ((uint32_t)in[4 * i + 1] << 8) - | ((uint32_t)in[4 * i + 2] << 16) - | ((uint32_t)in[4 * i + 3] << 24); - - strm->next_in += 4 * rsi; - strm->avail_in -= 4 * rsi; -} - -AEC_GET_RSI_NATIVE_32(msb); - -#else /* !WORDS_BIGENDIAN */ -void aec_get_rsi_msb_32(struct aec_stream *strm) -{ - uint32_t *restrict out = strm->state->data_raw; - const unsigned char *restrict in = strm->next_in; - int rsi = strm->rsi * strm->block_size; - - strm->next_in += 4 * rsi; - strm->avail_in -= 4 * rsi; - - for (int i = 0; i < rsi; i++) - out[i] = ((uint32_t)in[4 * i] << 24) - | ((uint32_t)in[4 * i + 1] << 16) - | ((uint32_t)in[4 * i + 2] << 8) - | (uint32_t)in[4 * i + 3]; -} - -AEC_GET_RSI_NATIVE_32(lsb) - -#endif /* !WORDS_BIGENDIAN */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.h deleted file mode 100644 index 5c345ac..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/encode_accessors.h +++ /dev/null @@ -1,73 +0,0 @@ -/** - * @file encode_accessors.h - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Read various data types from input stream - * - */ - -#ifndef ENCODE_ACCESSORS_H -#define ENCODE_ACCESSORS_H 1 - -#if HAVE_STDINT_H -# include -#endif - -uint32_t aec_get_8(struct aec_stream *strm); -uint32_t aec_get_lsb_16(struct aec_stream *strm); -uint32_t aec_get_msb_16(struct aec_stream *strm); -uint32_t aec_get_lsb_32(struct aec_stream *strm); -uint32_t aec_get_msb_24(struct aec_stream *strm); -uint32_t aec_get_lsb_24(struct aec_stream *strm); -uint32_t aec_get_msb_32(struct aec_stream *strm); - -void aec_get_rsi_8(struct aec_stream *strm); -void aec_get_rsi_lsb_16(struct aec_stream *strm); -void aec_get_rsi_msb_16(struct aec_stream *strm); -void aec_get_rsi_lsb_24(struct aec_stream *strm); -void aec_get_rsi_msb_24(struct aec_stream *strm); -void aec_get_rsi_lsb_32(struct aec_stream *strm); -void aec_get_rsi_msb_32(struct aec_stream *strm); - -#endif /* ENCODE_ACCESSORS_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/libaec.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/libaec.h deleted file mode 100644 index 4025532..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/libaec.h +++ /dev/null @@ -1,176 +0,0 @@ -/** - * @file libaec.h - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Adaptive Entropy Coding library - * - */ - -#ifndef LIBAEC_H -#define LIBAEC_H 1 - -#include - -#ifdef __cplusplus -extern "C"{ -#endif - -#if BUILDING_LIBAEC && HAVE_VISIBILITY -# define LIBAEC_DLL_EXPORTED __attribute__((__visibility__("default"))) -#elif BUILDING_LIBAEC && defined _MSC_VER -# define LIBAEC_DLL_EXPORTED __declspec(dllexport) -#elif defined _MSC_VER -# define LIBAEC_DLL_EXPORTED __declspec(dllimport) -#else -# define LIBAEC_DLL_EXPORTED -#endif - -struct internal_state; - -struct aec_stream { - const unsigned char *next_in; - - /* number of bytes available at next_in */ - size_t avail_in; - - /* total number of input bytes read so far */ - size_t total_in; - - unsigned char *next_out; - - /* remaining free space at next_out */ - size_t avail_out; - - /* total number of bytes output so far */ - size_t total_out; - - /* resolution in bits per sample (n = 1, ..., 32) */ - unsigned int bits_per_sample; - - /* block size in samples */ - unsigned int block_size; - - /* Reference sample interval, the number of Coded Data Sets - * between consecutive reference samples (up to 4096). */ - unsigned int rsi; - - unsigned int flags; - - struct internal_state *state; -}; - -/*********************************/ -/* Sample data description flags */ -/*********************************/ - -/* Samples are signed. Telling libaec this results in a slightly - * better compression ratio. Default is unsigned. */ -#define AEC_DATA_SIGNED 1 - -/* 24 bit samples are coded in 3 bytes */ -#define AEC_DATA_3BYTE 2 - -/* Samples are stored with their most significant bit first. This has - * nothing to do with the endianness of the host. Default is LSB. */ -#define AEC_DATA_MSB 4 - -/* Set if preprocessor should be used */ -#define AEC_DATA_PREPROCESS 8 - -/* Use restricted set of code options */ -#define AEC_RESTRICTED 16 - -/* Pad RSI to byte boundary. Only for decoding CCSDS sample data. */ -#define AEC_PAD_RSI 32 - -/* Do not enforce standard regarding legal block sizes. */ -#define AEC_NOT_ENFORCE 64 - -/*************************************/ -/* Return codes of library functions */ -/*************************************/ -#define AEC_OK 0 -#define AEC_CONF_ERROR (-1) -#define AEC_STREAM_ERROR (-2) -#define AEC_DATA_ERROR (-3) -#define AEC_MEM_ERROR (-4) - -/************************/ -/* Options for flushing */ -/************************/ - -/* Do not enforce output flushing. More input may be provided with - * later calls. So far only relevant for encoding. */ -#define AEC_NO_FLUSH 0 - -/* Flush output and end encoding. The last call to aec_encode() must - * set AEC_FLUSH to drain all output. - * - * It is not possible to continue encoding of the same stream after it - * has been flushed because the last byte may be padded with fill - * bits. */ -#define AEC_FLUSH 1 - -/*********************************************/ -/* Streaming encoding and decoding functions */ -/*********************************************/ -LIBAEC_DLL_EXPORTED int aec_encode_init(struct aec_stream *strm); -LIBAEC_DLL_EXPORTED int aec_encode(struct aec_stream *strm, int flush); -LIBAEC_DLL_EXPORTED int aec_encode_end(struct aec_stream *strm); - -LIBAEC_DLL_EXPORTED int aec_decode_init(struct aec_stream *strm); -LIBAEC_DLL_EXPORTED int aec_decode(struct aec_stream *strm, int flush); -LIBAEC_DLL_EXPORTED int aec_decode_end(struct aec_stream *strm); - -/***************************************************************/ -/* Utility functions for encoding or decoding a memory buffer. */ -/***************************************************************/ -LIBAEC_DLL_EXPORTED int aec_buffer_encode(struct aec_stream *strm); -LIBAEC_DLL_EXPORTED int aec_buffer_decode(struct aec_stream *strm); - -#ifdef __cplusplus -} -#endif - -#endif /* LIBAEC_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/sz_compat.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/sz_compat.c deleted file mode 100644 index f852ed2..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/sz_compat.c +++ /dev/null @@ -1,261 +0,0 @@ -#include -#include -#include -#if HAVE_CONFIG_H -# include -#endif -#include "szlib.h" - -#define NOPTS 129 -#define MIN(a, b) (((a) < (b))? (a): (b)) - -static int convert_options(int sz_opts) -{ - int co[NOPTS]; - int opts = 0; - - memset(co, 0, sizeof(int) * NOPTS); - co[SZ_MSB_OPTION_MASK] = AEC_DATA_MSB; - co[SZ_NN_OPTION_MASK] = AEC_DATA_PREPROCESS; - - for (int i = 1; i < NOPTS; i <<= 1) - if (sz_opts & i) - opts |= co[i]; - - return opts; -} - -static int bits_to_bytes(int bit_length) -{ - if (bit_length > 16) - return 4; - else if (bit_length > 8) - return 2; - else - return 1; -} - -static void interleave_buffer(void *dest, const void *src, - size_t n, int wordsize) -{ - const unsigned char *src8 = (unsigned char *)src; - unsigned char *dest8 = (unsigned char *)dest; - - for (size_t i = 0; i < n / wordsize; i++) - for (size_t j = 0; j < wordsize; j++) - dest8[j * (n / wordsize) + i] = src8[i * wordsize + j]; -} - -static void deinterleave_buffer(void *dest, const void *src, - size_t n, int wordsize) -{ - const unsigned char *src8 = (unsigned char *)src; - unsigned char *dest8 = (unsigned char *)dest; - - for (size_t i = 0; i < n / wordsize; i++) - for (size_t j = 0; j < wordsize; j++) - dest8[i * wordsize + j] = src8[j * (n / wordsize) + i]; -} - -static void add_padding(void *dest, const void *src, size_t src_length, - size_t line_size, size_t padding_size, - int pixel_size, int pp) -{ - const char zero_pixel[] = {0, 0, 0, 0}; - - const char *pixel = zero_pixel; - size_t j = 0; - size_t i = 0; - while (i < src_length) { - size_t ps; - size_t ls = MIN(src_length - i, line_size); - memcpy((char *)dest + j, (char *)src + i, ls); - j += ls; - i += ls; - if (pp) - pixel = (char *)src + i - pixel_size; - ps = line_size + padding_size - ls; - for (size_t k = 0; k < ps; k += pixel_size) - memcpy((char *)dest + j + k, pixel, pixel_size); - j += ps; - } -} - -static void remove_padding(void *buf, size_t buf_length, - size_t line_size, size_t padding_size, - int pixel_size) -{ - size_t padded_line_size = line_size + padding_size; - - size_t i = line_size; - for (size_t j = padded_line_size; j < buf_length; j += padded_line_size) { - memmove((char *)buf + i, (char *)buf + j, line_size); - i += line_size; - } -} - -int SZ_BufftoBuffCompress(void *dest, size_t *destLen, - const void *source, size_t sourceLen, - SZ_com_t *param) -{ - struct aec_stream strm; - void *buf = 0; - void *padbuf = 0; - int status; - int interleave; - int pixel_size; - int aec_status; - size_t scanlines; - size_t padbuf_size; - size_t padding_size; - - strm.block_size = param->pixels_per_block; - strm.rsi = (param->pixels_per_scanline + param->pixels_per_block - 1) - / param->pixels_per_block; - strm.flags = AEC_NOT_ENFORCE | convert_options(param->options_mask); - strm.avail_out = *destLen; - strm.next_out = dest; - - interleave = param->bits_per_pixel == 32 || param->bits_per_pixel == 64; - if (interleave) { - strm.bits_per_sample = 8; - buf = malloc(sourceLen); - if (buf == NULL) { - status = SZ_MEM_ERROR; - goto CLEANUP; - } - interleave_buffer(buf, source, sourceLen, param->bits_per_pixel / 8); - } else { - strm.bits_per_sample = param->bits_per_pixel; - buf = (void *)source; - } - - pixel_size = bits_to_bytes(strm.bits_per_sample); - - scanlines = (sourceLen / pixel_size + param->pixels_per_scanline - 1) - / param->pixels_per_scanline; - padbuf_size = strm.rsi * strm.block_size * pixel_size * scanlines; - padbuf = malloc(padbuf_size); - if (padbuf == NULL) { - status = SZ_MEM_ERROR; - goto CLEANUP; - } - - padding_size = - (strm.rsi * strm.block_size - param->pixels_per_scanline) - * pixel_size; - - add_padding(padbuf, buf, sourceLen, - param->pixels_per_scanline * pixel_size, - padding_size, pixel_size, - strm.flags & AEC_DATA_PREPROCESS); - strm.next_in = padbuf; - strm.avail_in = padbuf_size; - - aec_status = aec_buffer_encode(&strm); - if (aec_status == AEC_STREAM_ERROR) - status = SZ_OUTBUFF_FULL; - else - status = aec_status; - *destLen = strm.total_out; - -CLEANUP: - if (padbuf) - free(padbuf); - if (interleave && buf) - free(buf); - return status; -} - -int SZ_BufftoBuffDecompress(void *dest, size_t *destLen, - const void *source, size_t sourceLen, - SZ_com_t *param) -{ - struct aec_stream strm; - void *buf = 0; - int status; - int pad_scanline; - int deinterleave; - int extra_buffer; - int pixel_size; - size_t total_out; - size_t scanlines; - - strm.block_size = param->pixels_per_block; - strm.rsi = (param->pixels_per_scanline + param->pixels_per_block - 1) - / param->pixels_per_block; - strm.flags = convert_options(param->options_mask); - strm.avail_in = sourceLen; - strm.next_in = source; - - pad_scanline = param->pixels_per_scanline % param->pixels_per_block; - deinterleave = (param->bits_per_pixel == 32 - || param->bits_per_pixel == 64); - extra_buffer = pad_scanline || deinterleave; - - if (deinterleave) - strm.bits_per_sample = 8; - else - strm.bits_per_sample = param->bits_per_pixel; - pixel_size = bits_to_bytes(strm.bits_per_sample); - - - if (extra_buffer) { - size_t buf_size; - if (pad_scanline) { - scanlines = (*destLen / pixel_size + param->pixels_per_scanline - 1) - / param->pixels_per_scanline; - buf_size = strm.rsi * strm.block_size * pixel_size * scanlines; - } else { - buf_size = *destLen; - } - buf = malloc(buf_size); - if (buf == NULL) { - status = SZ_MEM_ERROR; - goto CLEANUP; - } - strm.next_out = buf; - strm.avail_out = buf_size; - } else { - strm.next_out = dest; - strm.avail_out = *destLen; - } - - status = aec_buffer_decode(&strm); - if (status != AEC_OK) - goto CLEANUP; - - if (pad_scanline) { - size_t padding_size = - (strm.rsi * strm.block_size - param->pixels_per_scanline) - * pixel_size; - remove_padding(buf, strm.total_out, - param->pixels_per_scanline * pixel_size, - padding_size, pixel_size); - total_out = scanlines * param->pixels_per_scanline * pixel_size; - } else { - total_out = strm.total_out; - } - - if (total_out < *destLen) - *destLen = total_out; - - if (deinterleave) - deinterleave_buffer(dest, buf, *destLen, param->bits_per_pixel / 8); - else if (pad_scanline) - memcpy(dest, buf, *destLen); - -CLEANUP: - if (extra_buffer && buf) - free(buf); - - return status; -} - -int SZ_encoder_enabled(void) -{ - return 1; -} - -/* netcdf searches for SZ_Compress in configure */ -char SZ_Compress() { return SZ_OK; } diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/szlib.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/szlib.h deleted file mode 100644 index c22362a..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/szlib.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef SZLIB_H -#define SZLIB_H 1 - -#include "libaec.h" - -#define SZ_ALLOW_K13_OPTION_MASK 1 -#define SZ_CHIP_OPTION_MASK 2 -#define SZ_EC_OPTION_MASK 4 -#define SZ_LSB_OPTION_MASK 8 -#define SZ_MSB_OPTION_MASK 16 -#define SZ_NN_OPTION_MASK 32 -#define SZ_RAW_OPTION_MASK 128 - -#define SZ_OK AEC_OK -#define SZ_OUTBUFF_FULL 2 - -#define SZ_NO_ENCODER_ERROR -1 -#define SZ_PARAM_ERROR AEC_CONF_ERROR -#define SZ_MEM_ERROR AEC_MEM_ERROR - -#define SZ_MAX_PIXELS_PER_BLOCK 32 -#define SZ_MAX_BLOCKS_PER_SCANLINE 128 -#define SZ_MAX_PIXELS_PER_SCANLINE \ - (SZ_MAX_BLOCKS_PER_SCANLINE) * (SZ_MAX_PIXELS_PER_BLOCK) - -typedef struct SZ_com_t_s -{ - int options_mask; - int bits_per_pixel; - int pixels_per_block; - int pixels_per_scanline; -} SZ_com_t; - -LIBAEC_DLL_EXPORTED int SZ_BufftoBuffCompress( - void *dest, size_t *destLen, - const void *source, size_t sourceLen, - SZ_com_t *param); -LIBAEC_DLL_EXPORTED int SZ_BufftoBuffDecompress( - void *dest, size_t *destLen, - const void *source, size_t sourceLen, - SZ_com_t *param); - -LIBAEC_DLL_EXPORTED int SZ_encoder_enabled(void); - -#endif /* SZLIB_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/utime.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/utime.c deleted file mode 100644 index a7f1e99..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/src/utime.c +++ /dev/null @@ -1,123 +0,0 @@ -/** - * @file utime.c - * - * @author Thomas Jahns, Deutsches Klimarechenzentrum - * - * @section LICENSE - * Copyright 2012 - 2017 - * - * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens - * Deutsches Klimarechenzentrum GmbH - * Bundesstr. 45a - * 20146 Hamburg Germany - * - * Luis Kornblueh - * Max-Planck-Institut fuer Meteorologie - * Bundesstr. 53 - * 20146 Hamburg - * Germany - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * @section DESCRIPTION - * - * Simple timing command, since calling time(1) gives non-portable results. - * - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -static int -run_cmd(int argc, char *argv[]); - -int main(int argc, char **argv) -{ - struct timeval utime = { .tv_sec = 0, .tv_usec = 0 }; - int rstatus; - int status = 0; - if (argc > 1 && ((status = run_cmd(argc - 1, argv + 1)) >= 0)) - { - struct rusage usage; - if ((rstatus = getrusage(RUSAGE_CHILDREN, &usage) == -1)) - { - perror("resource usage statistics unavailable"); - } - else if (rstatus != 0) - { - fputs("an unknown error occurred\n", stderr); - return EXIT_FAILURE; - } - else - { - utime = usage.ru_utime; - } - } - else if (status) - { - fputs("could not fork child\n", stderr); - return EXIT_FAILURE; - } - fprintf(stderr, "%f\n", (utime.tv_sec * 1000000 + utime.tv_usec)/1000000.0); - return status; -} - -static int -run_cmd(int argc, char *argv[]) -{ - int status; - pid_t child_pid; - if (argc < 1) - return -1; - if ((child_pid = fork()) < 0) - status = -1; - else if (child_pid == 0) - { - /* child */ - execvp(argv[0], argv); - _exit(127); /* execvp should not have returned */ - } - else - { - while (waitpid(child_pid, &status, 0) < 0) - if (errno != EINTR) - { - status = -1; - break; - } - } - return status; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/CMakeLists.txt b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/CMakeLists.txt deleted file mode 100644 index 3553ff5..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -add_library(check_aec STATIC check_aec.c) - -add_executable(check_code_options check_code_options.c) -target_link_libraries(check_code_options check_aec aec) -add_test(NAME check_code_options COMMAND check_code_options) -add_executable(check_buffer_sizes check_buffer_sizes.c) -target_link_libraries(check_buffer_sizes check_aec aec) -add_test(NAME check_buffer_sizes COMMAND check_buffer_sizes) -add_executable(check_long_fs check_long_fs.c) -target_link_libraries(check_long_fs check_aec aec) -add_test(NAME check_long_fs COMMAND check_long_fs) -add_executable(check_szcomp check_szcomp.c) -target_link_libraries(check_szcomp check_aec sz) -add_test(NAME check_szcomp - COMMAND check_szcomp ${PROJECT_SOURCE_DIR}/data/121B2TestData/ExtendedParameters/sar32bit.dat) - -if(UNIX) - add_test( - NAME sampledata.sh - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/sampledata.sh ${CMAKE_CURRENT_SOURCE_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -endif(UNIX) diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/Makefile.am b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/Makefile.am deleted file mode 100644 index eb62414..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/Makefile.am +++ /dev/null @@ -1,28 +0,0 @@ -AUTOMAKE_OPTIONS = color-tests -AM_CPPFLAGS = -I$(top_srcdir)/src -TESTS = check_code_options check_buffer_sizes check_long_fs \ -szcomp.sh sampledata.sh -TEST_EXTENSIONS = .sh -CLEANFILES = test.dat test.rz -check_LTLIBRARIES = libcheck_aec.la -libcheck_aec_la_SOURCES = check_aec.c check_aec.h -check_PROGRAMS = check_code_options check_buffer_sizes check_long_fs \ -check_szcomp - -check_code_options_SOURCES = check_code_options.c check_aec.h \ -$(top_srcdir)/src/libaec.h - -check_buffer_sizes_SOURCES = check_buffer_sizes.c check_aec.h \ -$(top_srcdir)/src/libaec.h - -check_long_fs_SOURCES = check_long_fs.c check_aec.h \ -$(top_srcdir)/src/libaec.h - -check_szcomp_SOURCES = check_szcomp.c $(top_srcdir)/src/szlib.h - -LDADD = libcheck_aec.la $(top_builddir)/src/libaec.la -check_szcomp_LDADD = $(top_builddir)/src/libsz.la - -EXTRA_DIST = sampledata.sh szcomp.sh CMakeLists.txt - -szcomp.log: sampledata.log diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.c deleted file mode 100644 index 354f357..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.c +++ /dev/null @@ -1,310 +0,0 @@ -#include -#include -#include -#include "check_aec.h" - -static void out_lsb(unsigned char *dest, unsigned long long int val, int size) -{ - for (int i = 0; i < size; i++) - dest[i] = (unsigned char)(val >> (8 * i)); -} - -static void out_msb(unsigned char *dest, unsigned long long int val, int size) -{ - for (int i = 0; i < size; i++) - dest[i] = (unsigned char)(val >> (8 * (size - 1 - i))); -} - -int update_state(struct test_state *state) -{ - struct aec_stream *strm = state->strm; - - if (strm->bits_per_sample > 16) { - state->id_len = 5; - - if (strm->bits_per_sample <= 24 && strm->flags & AEC_DATA_3BYTE) { - state->bytes_per_sample = 3; - } else { - state->bytes_per_sample = 4; - } - } - else if (strm->bits_per_sample > 8) { - state->id_len = 4; - state->bytes_per_sample = 2; - } else { - state->id_len = 3; - state->bytes_per_sample = 1; - } - - if (strm->flags & AEC_DATA_MSB) - state->out = out_msb; - else - state->out = out_lsb; - - if (strm->flags & AEC_DATA_SIGNED) { - state->xmin = -(1LL << (strm->bits_per_sample - 1)); - state->xmax = (1ULL << (strm->bits_per_sample - 1)) - 1; - } else { - state->xmin = 0; - state->xmax = (1ULL << strm->bits_per_sample) - 1; - } - - return 0; -} - -int encode_decode_small(struct test_state *state) -{ - size_t compressed_size; - size_t n_in, avail_in, avail_out, total_out; - struct aec_stream *strm = state->strm; - - int status = aec_encode_init(strm); - if (status != AEC_OK) { - printf("Init failed.\n"); - return 99; - } - - n_in = 0; - avail_in = 1; - avail_out = 1; - total_out = 0; - strm->next_in = state->ubuf; - strm->avail_in = state->bytes_per_sample; - strm->avail_out = 1; - strm->next_out = state->cbuf; - - while ((avail_in || avail_out) && total_out < state->cbuf_len) { - if (strm->avail_in == 0 && avail_in) { - n_in += state->bytes_per_sample; - if (n_in < state->buf_len) { - strm->avail_in = state->bytes_per_sample; - strm->next_in = state->ubuf + n_in; - } else { - avail_in = 0; - } - } - - status = aec_encode(strm, AEC_NO_FLUSH); - if (status != AEC_OK) { - printf("Decode failed.\n"); - return 99; - } - - if (strm->total_out - total_out > 0 - && total_out < state->cbuf_len) { - total_out = strm->total_out; - strm->avail_out = 1; - strm->next_out = state->cbuf + total_out; - avail_out = 1; - } else { - avail_out = 0; - } - } - - status = aec_encode(strm, AEC_FLUSH); - if (status != AEC_OK) { - printf("Encode failed.\n"); - return 99; - } - - aec_encode_end(strm); - - compressed_size = strm->total_out; - - strm->avail_in = 1; - strm->next_in = state->cbuf; - - strm->avail_out = state->bytes_per_sample; - strm->next_out = state->obuf; - - status = aec_decode_init(strm); - if (status != AEC_OK) { - printf("Init failed.\n"); - return 99; - } - - n_in = 0; - avail_in = 1; - avail_out = 1; - total_out = 0; - strm->next_in = state->cbuf; - strm->avail_in = 1; - strm->avail_out = state->bytes_per_sample; - strm->next_out = state->obuf; - - while ((avail_in || avail_out) && total_out < state->buf_len) { - if (strm->avail_in == 0 && avail_in) { - n_in++; - if (n_in < compressed_size) { - strm->avail_in = 1; - strm->next_in = state->cbuf + n_in; - } else { - avail_in = 0; - } - } - - status = aec_decode(strm, AEC_NO_FLUSH); - if (status != AEC_OK) { - printf("Decode failed.\n"); - return 99; - } - - if (strm->total_out - total_out > 0 - && total_out < state->buf_len) { - total_out = strm->total_out; - strm->avail_out = state->bytes_per_sample; - strm->next_out = state->obuf + total_out; - avail_out = 1; - } else { - avail_out = 0; - } - } - - status = aec_decode(strm, AEC_FLUSH); - if (status != AEC_OK) { - printf("Decode failed.\n"); - return 99; - } - - if (memcmp(state->ubuf, state->obuf, state->ibuf_len)) { - printf("\n%s: Uncompressed output differs from input.\n", CHECK_FAIL); - - printf("\nuncompressed buf"); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n"); - printf("%02x ", state->ubuf[i]); - } - printf("\n\ncompressed buf len %zu", compressed_size); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n"); - printf("%02x ", state->cbuf[i]); - } - printf("\n\ndecompressed buf"); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n%04i ", i); - printf("%02x ", state->obuf[i]); - } - printf("\n"); - return 99; - } - aec_decode_end(strm); - return 0; -} - -int encode_decode_large(struct test_state *state) -{ - int status; - int bflags = 0; - size_t to; - char fbase[1024]; - struct aec_stream *strm = state->strm; - - strm->avail_in = state->ibuf_len; - strm->avail_out = state->cbuf_len; - strm->next_in = state->ubuf; - strm->next_out = state->cbuf; - - status = aec_encode_init(strm); - if (status != AEC_OK) { - printf("Init failed.\n"); - return 99; - } - if (state->dump) { - char fname[1024]; - FILE *fp; - snprintf(fbase, sizeof(fbase), "BPS%02iID%iBS%02iRSI%04iFLG%04i", - strm->bits_per_sample, - state->id, - strm->block_size, - strm->rsi, - strm->flags); - snprintf(fname, sizeof(fname), "%s.dat", fbase); - if ((fp = fopen(fname, "wb")) == NULL) { - fprintf(stderr, "ERROR: cannot open dump file %s\n", fname); - return 99; - } - fputc(strm->bits_per_sample, fp); - bflags = strm->block_size >> 8; - if (strm->flags | AEC_DATA_MSB) - bflags |= 0x80; - if (strm->flags | AEC_DATA_SIGNED) - bflags |= 0x40; - if (strm->flags | AEC_DATA_3BYTE) - bflags |= 0x10; - bflags |= 0x20; /* encode */ - fputc(bflags, fp); - fwrite(strm->next_in, strm->avail_in, 1, fp); - fclose(fp); - } - - status = aec_encode(strm, AEC_FLUSH); - if (status != AEC_OK) { - printf("Encode failed.\n"); - return 99; - } - - aec_encode_end(strm); - - if (state->dump) { - char fname[1024]; - FILE *fp; - snprintf(fname, sizeof(fname), "%s.rz", fbase); - if ((fp = fopen(fname, "wb")) == NULL) { - fprintf(stderr, "ERROR: cannot open dump file %s\n", fname); - return 99; - } - fputc(strm->bits_per_sample, fp); - bflags &= ~0x20; - fputc(bflags, fp); - fwrite(state->cbuf, strm->total_out, 1, fp); - fclose(fp); - } - - strm->avail_in = strm->total_out; - strm->avail_out = state->buf_len; - strm->next_in = state->cbuf; - strm->next_out = state->obuf; - to = strm->total_out; - - status = aec_decode_init(strm); - if (status != AEC_OK) { - printf("Init failed.\n"); - return 99; - } - - status = aec_decode(strm, AEC_FLUSH); - if (status != AEC_OK) { - printf("Decode failed.\n"); - return 99; - } - - if (memcmp(state->ubuf, state->obuf, state->ibuf_len)) { - printf("\n%s: Uncompressed output differs from input.\n", CHECK_FAIL); - - printf("\nuncompressed buf"); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n"); - printf("%02x ", state->ubuf[i]); - } - printf("\n\ncompressed buf len %zu", to); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n"); - printf("%02x ", state->cbuf[i]); - } - printf("\n\ndecompressed buf"); - for (int i = 0; i < 80; i++) { - if (i % 8 == 0) - printf("\n"); - printf("%02x ", state->obuf[i]); - } - printf("\n"); - return 99; - } - aec_decode_end(strm); - return 0; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.h b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.h deleted file mode 100644 index 2f6abd3..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_aec.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef CHECK_AEC_H -#define CHECK_AEC_H 1 - -#include -#include "libaec.h" - -struct test_state { - int (* codec)(struct test_state *state); - int id; - int id_len; - int bytes_per_sample; - unsigned char *ubuf; - unsigned char *cbuf; - unsigned char *obuf; - size_t ibuf_len; /* input buffer legth may be shorter than buf_len */ - size_t buf_len; - size_t cbuf_len; - long long int xmax; - long long int xmin; - void (*out)(unsigned char *dest, unsigned long long int val, int size); - int dump; /* dump buffer to file for fuzzing corpus */ - struct aec_stream *strm; -}; - -int update_state(struct test_state *state); -int encode_decode_small(struct test_state *state); -int encode_decode_large(struct test_state *state); - -#ifndef HAVE_SNPRINTF -#ifdef HAVE__SNPRINTF_S -#define snprintf(d, n, ...) _snprintf_s((d), (n), _TRUNCATE, __VA_ARGS__) -#else -#ifdef HAVE__SNPRINTF -#define snprintf _snprintf -#else -#error "no snprintf compatible function found" -#endif /* HAVE__SNPRINTF */ -#endif /* HAVE__SNPRINTF_S */ -#endif /* HAVE_SNPRINTF */ - -#ifdef _WIN32 -#define CHECK_PASS "PASS" -#define CHECK_FAIL "FAIL" -#else -#define CHECK_PASS "PASS" -#define CHECK_FAIL "FAIL" -#endif - -#endif /* CHECK_AEC_H */ diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_buffer_sizes.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_buffer_sizes.c deleted file mode 100644 index 53996a0..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_buffer_sizes.c +++ /dev/null @@ -1,108 +0,0 @@ -#include -#include -#include -#include "check_aec.h" - -#define BUF_SIZE 1024 * 3 - -int check_block_sizes(struct test_state *state) -{ - for (int bs = 8; bs <= 64; bs *= 2) { - int status; - state->strm->block_size = bs; - state->strm->rsi = (int)(state->buf_len - / (bs * state->bytes_per_sample)); - - status = encode_decode_large(state); - if (status) - return status; - } - return 0; -} - -int check_block_sizes_short(struct test_state *state) -{ - size_t tmp = state->ibuf_len; - for (int bs = 8; bs <= 64; bs *= 2) { - int status; - state->strm->block_size = bs; - state->strm->rsi = (int)(state->buf_len - / (bs * state->bytes_per_sample)); - state->ibuf_len = state->buf_len - 2 * bs + 4; - status = encode_decode_large(state); - if (status) - return status; - if (state->strm->total_out != state->buf_len) { - printf("FAIL: Unexpected buffer length. Got %i expected %i\n", - (int)state->strm->total_out, - (int)state->buf_len); - return 99; - } - } - state->ibuf_len = tmp; - return 0; -} - -int check_rsi(struct test_state *state) -{ - int status; - int size = state->bytes_per_sample; - - for (unsigned char *tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 2 * state->bytes_per_sample) { - state->out(tmp, state->xmax, size); - state->out(tmp + size, state->xmin, size); - } - - printf("Checking full rsi ... "); - status = check_block_sizes(state); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - - printf("Checking short rsi ... "); - status = check_block_sizes_short(state); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int main (void) -{ - int status; - struct aec_stream strm; - struct test_state state; - - state.dump = 0; - state.buf_len = state.ibuf_len = BUF_SIZE; - state.cbuf_len = 2 * BUF_SIZE; - - state.ubuf = (unsigned char *)malloc(state.buf_len); - state.cbuf = (unsigned char *)malloc(state.cbuf_len); - state.obuf = (unsigned char *)malloc(state.buf_len); - - if (!state.ubuf || !state.cbuf || !state.obuf) { - printf("Not enough memory.\n"); - return 99; - } - - strm.flags = AEC_DATA_PREPROCESS; - state.strm = &strm; - strm.bits_per_sample = 32; - update_state(&state); - - status = check_rsi(&state); - if (status) - goto DESTRUCT; - -DESTRUCT: - free(state.ubuf); - free(state.cbuf); - free(state.obuf); - - return status; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_code_options.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_code_options.c deleted file mode 100644 index dea916f..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_code_options.c +++ /dev/null @@ -1,332 +0,0 @@ -#include -#include -#include -#include "check_aec.h" - -#define BUF_SIZE 1024 * 3 - -int check_block_sizes(struct test_state *state, int id, int id_len) -{ - state->id = id; - for (int bs = 8; bs <= 64; bs *= 2) { - int max_rsi; - - state->strm->block_size = bs; - max_rsi = (int)(state->buf_len / (bs * state->bytes_per_sample)); - if (max_rsi > 4096) - max_rsi = 4096; - - for (int rsi = 1; rsi <= max_rsi; rsi++) { - int status; - state->strm->rsi = rsi; - status = state->codec(state); - if (status) - return status; - - if ((state->cbuf[0] >> (8 - id_len)) != id) { - printf( - "%s: block of size %i created with ID:%x, expected %x.\n", - CHECK_FAIL, bs, state->cbuf[0] >> (8 - id_len), id - ); - return 99; - } - } - } - return 0; -} - -int check_zero(struct test_state *state) -{ - int status; - - if (state->strm->flags & AEC_DATA_PREPROCESS) - memset(state->ubuf, 0x55, state->buf_len); - else - memset(state->ubuf, 0, state->buf_len); - - printf("Checking zero blocks ... "); - status = check_block_sizes(state, 0, state->id_len + 1); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int check_splitting(struct test_state *state, int k) -{ - int status; - unsigned char *tmp; - - int size = state->bytes_per_sample; - - if (state->strm->flags & AEC_DATA_PREPROCESS) { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 4 * size) { - state->out(tmp, state->xmin + (1ULL << (k - 1)) - 1, size); - state->out(tmp + size, state->xmin, size); - state->out(tmp + 2 * size, state->xmin - + (1ULL << (k + 1)) - 1, size); - state->out(tmp + 3 * size, state->xmin, size); - } - } else { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 4 * size) { - state->out(tmp, 0, size); - state->out(tmp + size, (1ULL << k) - 1, size); - state->out(tmp + 2 * size, 0, size); - state->out(tmp + 3 * size, (1ULL << (k + 2)) - 1, size); - } - } - - printf("Checking splitting with k=%i ... ", k); - status = check_block_sizes(state, k + 1, state->id_len); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int check_uncompressed(struct test_state *state) -{ - int status; - unsigned char *tmp; - - int size = state->bytes_per_sample; - - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 2 * size) { - state->out(tmp, state->xmax, size); - state->out(tmp + size, state->xmin, size); - } - - printf("Checking uncompressed ... "); - status = check_block_sizes(state, - (1ULL << state->id_len) - 1, - state->id_len); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int check_fs(struct test_state *state) -{ - int status; - unsigned char *tmp; - - int size = state->bytes_per_sample; - - if (state->strm->flags & AEC_DATA_PREPROCESS) { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 4 * size) { - state->out(tmp, state->xmin + 2, size); - state->out(tmp + size, state->xmin, size); - state->out(tmp + 2 * size, state->xmin, size); - state->out(tmp + 3 * size, state->xmin, size); - } - } else { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 4 * size) { - state->out(tmp, 0, size); - state->out(tmp + size, 0, size); - state->out(tmp + 2 * size, 0, size); - state->out(tmp + 3 * size, 4, size); - } - } - - printf("Checking FS ... "); - status = check_block_sizes(state, 1, state->id_len); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int check_se(struct test_state *state) -{ - int status; - unsigned char *tmp; - - int size = state->bytes_per_sample; - - if (state->strm->flags & AEC_DATA_PREPROCESS) { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 8 * size) { - state->out(tmp, state->xmax - 1, size); - state->out(tmp + size, state->xmax - 1, size); - state->out(tmp + 2 * size, state->xmax - 1, size); - state->out(tmp + 3 * size, state->xmax - 1, size); - state->out(tmp + 4 * size, state->xmax, size); - state->out(tmp + 5 * size, state->xmax, size); - state->out(tmp + 6 * size, state->xmax, size); - state->out(tmp + 7 * size, state->xmax, size); - } - } else { - for (tmp = state->ubuf; - tmp < state->ubuf + state->buf_len; - tmp += 8 * size) { - state->out(tmp, 0, size); - state->out(tmp + size, 0, size); - state->out(tmp + 2 * size, 0, size); - state->out(tmp + 3 * size, 0, size); - state->out(tmp + 4 * size, 1, size); - state->out(tmp + 5 * size, 0, size); - state->out(tmp + 6 * size, 0, size); - state->out(tmp + 7 * size, 2, size); - } - } - - printf("Checking Second Extension ... "); - status = check_block_sizes(state, 1, state->id_len + 1); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int check_bps(struct test_state *state) -{ - int status; - - for (int bps = 8; bps <= 32; bps += 8) { - state->strm->bits_per_sample = bps; - if (bps == 24) - state->strm->flags |= AEC_DATA_3BYTE; - else - state->strm->flags &= ~AEC_DATA_3BYTE; - - update_state(state); - - status = check_zero(state); - if (status) - return status; - - status = check_se(state); - if (status) - return status; - - status = check_uncompressed(state); - if (status) - return status; - - status = check_fs(state); - if (status) - return status; - - for (int k = 1; k < bps - 2; k++) { - status = check_splitting(state, k); - if (status) - return status; - } - printf("All checks with %i bit per sample passed.\n", bps); - } - return 0; -} - -int check_byte_orderings(struct test_state *state) -{ - int status; - - printf("-----------------------------------\n"); - printf("Checking no PP, LSB first, unsigned\n"); - printf("-----------------------------------\n"); - status = check_bps(state); - if (status) - return status; - - printf("-----------------------------------\n"); - printf("Checking PP, LSB first, unsigned\n"); - printf("-----------------------------------\n"); - state->strm->flags |= AEC_DATA_PREPROCESS; - status = check_bps(state); - if (status) - return status; - - printf("-----------------------------------\n"); - printf("Checking PP, LSB first, signed\n"); - printf("-----------------------------------\n"); - state->strm->flags |= AEC_DATA_SIGNED; - - status = check_bps(state); - if (status) - return status; - - state->strm->flags &= ~AEC_DATA_SIGNED; - state->strm->flags |= AEC_DATA_MSB; - - printf("-----------------------------------\n"); - printf("Checking PP, MSB first, unsigned\n"); - printf("-----------------------------------\n"); - status = check_bps(state); - if (status) - return status; - - printf("-----------------------------------\n"); - printf("Checking PP, MSB first, signed\n"); - printf("-----------------------------------\n"); - state->strm->flags |= AEC_DATA_SIGNED; - - status = check_bps(state); - if (status) - return status; - return 0; -} - -int main(int argc, char *argv[]) -{ - int status; - struct aec_stream strm; - struct test_state state; - - if (argc > 1 && strncmp(argv[1], "-d", 2) == 0) - state.dump = 1; - else - state.dump = 0; - - state.buf_len = state.ibuf_len = BUF_SIZE; - state.cbuf_len = 2 * BUF_SIZE; - - state.ubuf = (unsigned char *)malloc(state.buf_len); - state.cbuf = (unsigned char *)malloc(state.cbuf_len); - state.obuf = (unsigned char *)malloc(state.buf_len); - - if (!state.ubuf || !state.cbuf || !state.obuf) { - printf("Not enough memory.\n"); - return 99; - } - - strm.flags = 0; - state.strm = &strm; - - printf("***************************\n"); - printf("Checking with small buffers\n"); - printf("***************************\n"); - state.codec = encode_decode_small; - status = check_byte_orderings(&state); - if (status) - goto DESTRUCT; - - printf("***************************\n"); - printf("Checking with large buffers\n"); - printf("***************************\n"); - state.codec = encode_decode_large; - status = check_byte_orderings(&state); - -DESTRUCT: - free(state.ubuf); - free(state.cbuf); - free(state.obuf); - - return status; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_long_fs.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_long_fs.c deleted file mode 100644 index 49bdaa3..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_long_fs.c +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include "check_aec.h" - -#define BUF_SIZE (64 * 4) - -int check_long_fs(struct test_state *state) -{ - int status; - int size = state->bytes_per_sample; - int bs = state->strm->block_size; - - for (int i = 0; i < bs / 2; i++) { - state->out(state->ubuf + size * i, state->xmin, size); - state->out(state->ubuf + bs * size / 2 + size * i, 65000, size); - } - - printf("Checking long fs ... "); - - status = state->codec(state); - if (status) - return status; - - printf ("%s\n", CHECK_PASS); - return 0; -} - -int main (void) -{ - int status; - struct aec_stream strm; - struct test_state state; - - state.dump = 0; - state.buf_len = state.ibuf_len = BUF_SIZE; - state.cbuf_len = 2 * BUF_SIZE; - - state.ubuf = (unsigned char *)malloc(state.buf_len); - state.cbuf = (unsigned char *)malloc(state.cbuf_len); - state.obuf = (unsigned char *)malloc(state.buf_len); - - if (!state.ubuf || !state.cbuf || !state.obuf) { - printf("Not enough memory.\n"); - return 99; - } - - strm.flags = AEC_DATA_PREPROCESS; - state.strm = &strm; - strm.bits_per_sample = 16; - strm.block_size = 64; - strm.rsi = 1; - state.codec = encode_decode_large; - update_state(&state); - - status = check_long_fs(&state); - if (status) - goto DESTRUCT; - -DESTRUCT: - free(state.ubuf); - free(state.cbuf); - free(state.obuf); - - return status; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_szcomp.c b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_szcomp.c deleted file mode 100644 index 8133754..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/check_szcomp.c +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include "szlib.h" - -#define OPTIONS_MASK (SZ_RAW_OPTION_MASK \ - | SZ_MSB_OPTION_MASK \ - | SZ_NN_OPTION_MASK) -#define PIXELS_PER_BLOCK (8) -#define PIXELS_PER_SCANLINE (PIXELS_PER_BLOCK*128) - -int main(int argc, char *argv[]) -{ - int status; - SZ_com_t sz_param; - unsigned char *source, *dest, *dest1; - size_t destLen, dest1Len, sourceLen; - FILE *fp; - - if (argc < 2) { - fprintf(stderr, "Usage: %s file\n", argv[0]); - return 1; - } - - if ((fp = fopen(argv[1], "rb")) == NULL) { - fprintf(stderr, "Can't open %s\n", argv[1]); - return 1; - } - fseek(fp, 0L, SEEK_END); - sourceLen = ftell(fp); - fseek(fp, 0L, SEEK_SET); - destLen = sourceLen + sourceLen / 10; - - sz_param.options_mask = OPTIONS_MASK; - sz_param.bits_per_pixel = 64; - sz_param.pixels_per_block = PIXELS_PER_BLOCK; - sz_param.pixels_per_scanline = PIXELS_PER_SCANLINE; - - source = (unsigned char *)malloc(sourceLen); - dest = (unsigned char *)malloc(destLen); - dest1 = (unsigned char *)malloc(destLen); - - if (source == NULL || dest == NULL || dest1 == NULL) - return 1; - - sourceLen = fread(source, 1, sourceLen, fp); - - status = SZ_BufftoBuffCompress(dest, &destLen, - source, sourceLen, &sz_param); - if (status != SZ_OK) - return status; - - dest1Len = sourceLen; - status = SZ_BufftoBuffDecompress(dest1, &dest1Len, - dest, destLen, &sz_param); - if (status != SZ_OK) - return status; - - if (memcmp(source, dest1, sourceLen) != 0) - fprintf(stderr, "File %s Buffers differ\n", argv[2]); - - free(source); - free(dest); - free(dest1); - return 0; -} diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/sampledata.sh b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/sampledata.sh deleted file mode 100755 index 5729c8e..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/sampledata.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh -set -e -AEC=../src/aec -if [ -n "$1" ]; then - srcdir=$1 -fi -CCSDS_DATA=${srcdir}/../data/121B2TestData -ALLO=${CCSDS_DATA}/AllOptions -EXTP=${CCSDS_DATA}/ExtendedParameters -LOWE=${CCSDS_DATA}/LowEntropyOptions - -filesize () { - wc -c $1 | awk '{print $1}' -} - -decode () { - $AEC -d $3 $1 test.dat - dd if=test.dat bs=1 count=$(filesize $2) | cmp $2 - -} - -code () { - $AEC $3 $2 test.rz - cmp $1 test.rz -} - -code_size () { - $AEC $3 $2 test.rz - if [ ! $(filesize test.rz) -eq $(filesize $1) ]; then - echo "$1 size mismatch" - exit 1 - fi -} - -codec () { - code "$@" - decode "$@" -} - -cosdec () { - code_size "$@" - decode "$@" -} - -echo All Options -for i in 01 02 03 04 -do - uf=$ALLO/test_p256n${i}.dat - codec $ALLO/test_p256n${i}-basic.rz $uf "-n$i -j16 -r16" - codec $ALLO/test_p256n${i}-restricted.rz $uf "-n$i -j16 -r16 -t" -done -for i in 05 06 07 08 09 10 11 12 13 14 15 16 -do - cosdec $ALLO/test_p256n${i}.rz $ALLO/test_p256n${i}.dat \ - "-n$i -j16 -r16" -done -for i in 17 18 19 20 21 22 23 24 -do - cosdec $ALLO/test_p512n${i}.rz $ALLO/test_p512n${i}.dat \ - "-n$i -j16 -r32" -done - -echo Low Entropy Options -for i in 1 2 3 -do - for j in 01 02 03 04 - do - uf=$LOWE/Lowset${i}_8bit.dat - codec $LOWE/Lowset${i}_8bit.n${j}-basic.rz $uf "-n$j -j16 -r64" - codec $LOWE/Lowset${i}_8bit.n${j}-restricted.rz $uf "-n$j -j16 -r64 -t" - done - for j in 05 06 07 08 - do - codec $LOWE/Lowset${i}_8bit.n${j}.rz $LOWE/Lowset${i}_8bit.dat \ - "-n$j -j16 -r64" - done -done - -echo Extended Parameters -decode $EXTP/sar32bit.j16.r256.rz $EXTP/sar32bit.dat "-n32 -j16 -r256 -p" -decode $EXTP/sar32bit.j64.r4096.rz $EXTP/sar32bit.dat "-n32 -j64 -r4096 -p" diff --git a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/szcomp.sh b/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/szcomp.sh deleted file mode 100755 index f140932..0000000 --- a/libaec-v1.0.2-b428510436e86c34908e3cb00a6b2e62aa1390da/tests/szcomp.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e -testfile=${srcdir}/../data/121B2TestData/ExtendedParameters/sar32bit.dat -if [ ! -f $testfile ]; then - echo "ERROR: sample data not found." - exit -1 -fi -./check_szcomp $testfile diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..93c681b --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,52 @@ +add_library(aec ${LIB_TYPE} ${libaec_SRCS}) +set_target_properties(aec PROPERTIES VERSION 0.0.8 SOVERSION 0) +add_library(sz ${LIB_TYPE} sz_compat.c) +set_target_properties(sz PROPERTIES VERSION 2.0.1 SOVERSION 2) + +target_link_libraries(sz aec) +if(WIN32 AND BUILD_SHARED_LIBS) + set_target_properties (aec PROPERTIES DEFINE_SYMBOL "BUILDING_LIBAEC") + set_target_properties (sz PROPERTIES DEFINE_SYMBOL "BUILDING_LIBAEC") + set_target_properties(sz PROPERTIES OUTPUT_NAME "szip") +endif(WIN32 AND BUILD_SHARED_LIBS) + +add_executable(aec_client aec.c) +set_target_properties(aec_client PROPERTIES OUTPUT_NAME "aec") +target_link_libraries(aec_client aec) + +if(UNIX) + add_executable(utime EXCLUDE_FROM_ALL utime.c) + add_custom_target(bench + COMMAND ${CMAKE_SOURCE_DIR}/src/benc.sh ${CMAKE_SOURCE_DIR}/data/typical.rz + COMMAND ${CMAKE_SOURCE_DIR}/src/bdec.sh + DEPENDS aec_client utime) +endif(UNIX) + +include(GNUInstallDirs) +if(WIN32) + install(TARGETS aec sz + RUNTIME + DESTINATION bin + ARCHIVE + DESTINATION lib + COMPONENT libraries) +else(WIN32) + install(TARGETS aec sz + LIBRARY + DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE + DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT libraries) + install(FILES aec.1 + DESTINATION ${CMAKE_INSTALL_FULL_MANDIR}/man1 COMPONENT doc) +endif(WIN32) + +install(FILES libaec.h szlib.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT headers) +install(TARGETS aec_client + RUNTIME + DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT applications) + +set(CPACK_COMPONENTS_ALL applications libraries headers doc) diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..de81372 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,28 @@ +AM_CFLAGS = $(CFLAG_VISIBILITY) +AM_CPPFLAGS = -DBUILDING_LIBAEC +lib_LTLIBRARIES = libaec.la libsz.la +libaec_la_SOURCES = encode.c encode_accessors.c decode.c \ +encode.h encode_accessors.h decode.h +libaec_la_LDFLAGS = -version-info 0:8:0 -no-undefined + +libsz_la_SOURCES = sz_compat.c +libsz_la_LIBADD = libaec.la +libsz_la_LDFLAGS = -version-info 2:1:0 -no-undefined + +include_HEADERS = libaec.h szlib.h + +bin_PROGRAMS = aec +noinst_PROGRAMS = utime +utime_SOURCES = utime.c +aec_LDADD = libaec.la +aec_SOURCES = aec.c +dist_man_MANS = aec.1 + +EXTRA_DIST = CMakeLists.txt benc.sh bdec.sh +CLEANFILES = bench.dat bench.rz + +bench-local: all benc bdec +benc-local: all + $(srcdir)/benc.sh $(top_srcdir)/data/typical.rz +bdec-local: all + top_srcdir=$(top_srcdir) $(srcdir)/bdec.sh diff --git a/src/aec.1 b/src/aec.1 new file mode 100644 index 0000000..fa7dc1f --- /dev/null +++ b/src/aec.1 @@ -0,0 +1,57 @@ +.TH AEC 1 +.SH NAME +aec \- compress or expand files +.SH SYNOPSIS +.B aec +[\fB\-3\fR] +[\fB\-b\fR \fIBYTES\fR] +[\fB\-d\fR] +[\fB\-j\fR \fISAMPLES\fR] +[\fB\-m\fR] +[\fB\-n\fR \fIBITS\fR] +[\fB\-N\fR] +[\fB\-p\fR] +[\fB\-r\fR \fIBLOCKS\fR] +[\fB\-s\fR] +[\fB\-t\fR] +.IR infile +.IR outfile +.SH DESCRIPTION +.IR Aec +performs lossless compression and decompression with Golomb-Rice coding +as defined in the Space Data System Standard documents 121.0-B-2. +.SH OPTIONS +.TP +\fB \-3\fR +24 bit samples are stored in 3 bytes +.TP +\fB \-b\fR\ \fI\,BYTES\fR +internal buffer size in bytes +.TP +\fB \-d\fR +decompress \fIinfile\fR; if option \-d is not used then compress +\fIinfile\fR +.TP +\fB \-j\fR \fI\,SAMPLES\fR +block size in samples +.TP +\fB \-m\fR +samples are MSB first; default is LSB first +.TP +\fB \-n\fR \fI\,BITS\fR +bits per sample +.TP +\fB \-N\fR +disable pre/post processing +.TP +\fB \-p\fR +pad RSI to byte boundary +.TP +\fB \-r\fR \fI\,BLOCKS\fR +reference sample interval in blocks +.TP +\fB \-s\fR +samples are signed; default is unsigned +.TP +\fB \-t\fR +use restricted set of code options diff --git a/src/aec.c b/src/aec.c new file mode 100644 index 0000000..b3c6a36 --- /dev/null +++ b/src/aec.c @@ -0,0 +1,258 @@ +/** + * @file aec.c + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * CLI frontend for Adaptive Entropy Coding library + * + */ + +#include +#include +#include + +#include "libaec.h" + +#define CHUNK 10485760 + +int get_param(unsigned int *param, int *iarg, char *argv[]) +{ + if (strlen(argv[*iarg]) == 2) { + (*iarg)++; + if (argv[*iarg][0] == '-') + return 1; + else + *param = atoi(argv[*iarg]); + } else { + *param = atoi(&argv[*iarg][2]); + } + return 0; +} + +int main(int argc, char *argv[]) +{ + struct aec_stream strm; + unsigned char *in; + unsigned char *out; + size_t total_out; + unsigned int chunk; + int status; + int input_avail, output_avail; + char *infn, *outfn; + FILE *infp, *outfp; + int dflag; + char *opt; + int iarg; + + chunk = CHUNK; + strm.bits_per_sample = 8; + strm.block_size = 8; + strm.rsi = 2; + strm.flags = AEC_DATA_PREPROCESS; + dflag = 0; + iarg = 1; + + while (iarg < argc - 2) { + opt = argv[iarg]; + if (opt[0] != '-') + goto FAIL; + switch (opt[1]) { + case '3': + strm.flags |= AEC_DATA_3BYTE; + break; + case 'N': + strm.flags &= ~AEC_DATA_PREPROCESS; + break; + case 'b': + if (get_param(&chunk, &iarg, argv)) + goto FAIL; + break; + case 'd': + dflag = 1; + break; + case 'j': + if (get_param(&strm.block_size, &iarg, argv)) + goto FAIL; + break; + case 'm': + strm.flags |= AEC_DATA_MSB; + break; + case 'n': + if (get_param(&strm.bits_per_sample, &iarg, argv)) + goto FAIL; + break; + case 'p': + strm.flags |= AEC_PAD_RSI; + break; + case 'r': + if (get_param(&strm.rsi, &iarg, argv)) + goto FAIL; + break; + case 's': + strm.flags |= AEC_DATA_SIGNED; + break; + case 't': + strm.flags |= AEC_RESTRICTED; + break; + default: + goto FAIL; + } + iarg++; + } + + if (argc - iarg < 2) + goto FAIL; + + infn = argv[iarg]; + outfn = argv[iarg + 1]; + + if (strm.bits_per_sample > 16) { + if (strm.bits_per_sample <= 24 && strm.flags & AEC_DATA_3BYTE) + chunk *= 3; + else + chunk *= 4; + } else if (strm.bits_per_sample > 8) { + chunk *= 2; + } + + out = (unsigned char *)malloc(chunk); + in = (unsigned char *)malloc(chunk); + + if (in == NULL || out == NULL) + exit(-1); + + total_out = 0; + strm.avail_in = 0; + strm.avail_out = chunk; + strm.next_out = out; + + input_avail = 1; + output_avail = 1; + + if ((infp = fopen(infn, "rb")) == NULL) { + fprintf(stderr, "ERROR: cannot open input file %s\n", infn); + return 1; + } + if ((outfp = fopen(outfn, "wb")) == NULL) { + fprintf(stderr, "ERROR: cannot open output file %s\n", infn); + return 1; + } + + if (dflag) + status = aec_decode_init(&strm); + else + status = aec_encode_init(&strm); + + if (status != AEC_OK) { + fprintf(stderr, "ERROR: initialization failed (%d)\n", status); + return 1; + } + + while(input_avail || output_avail) { + if (strm.avail_in == 0 && input_avail) { + strm.avail_in = fread(in, 1, chunk, infp); + if (strm.avail_in != chunk) + input_avail = 0; + strm.next_in = in; + } + + if (dflag) + status = aec_decode(&strm, AEC_NO_FLUSH); + else + status = aec_encode(&strm, AEC_NO_FLUSH); + + if (status != AEC_OK) { + fprintf(stderr, "ERROR: %i\n", status); + return 1; + } + + if (strm.total_out - total_out > 0) { + fwrite(out, strm.total_out - total_out, 1, outfp); + total_out = strm.total_out; + output_avail = 1; + strm.next_out = out; + strm.avail_out = chunk; + } else { + output_avail = 0; + } + + } + + if (dflag) { + aec_decode_end(&strm); + } else { + if ((status = aec_encode(&strm, AEC_FLUSH)) != AEC_OK) { + fprintf(stderr, "ERROR: while flushing output (%i)\n", status); + return 1; + } + + if (strm.total_out - total_out > 0) + fwrite(out, strm.total_out - total_out, 1, outfp); + + aec_encode_end(&strm); + } + + fclose(infp); + fclose(outfp); + free(in); + free(out); + return 0; + +FAIL: + fprintf(stderr, "NAME\n\taec - encode or decode files "); + fprintf(stderr, "with Adaptive Entropy Coding\n\n"); + fprintf(stderr, "SYNOPSIS\n\taec [OPTION]... SOURCE DEST\n"); + fprintf(stderr, "\nOPTIONS\n"); + fprintf(stderr, "\t-3\n\t\t24 bit samples are stored in 3 bytes\n"); + fprintf(stderr, "\t-N\n\t\tdisable pre/post processing\n"); + fprintf(stderr, "\t-b size\n\t\tinternal buffer size in bytes\n"); + fprintf(stderr, "\t-d\n\t\tdecode SOURCE. If -d is not used: encode.\n"); + fprintf(stderr, "\t-j samples\n\t\tblock size in samples\n"); + fprintf(stderr, "\t-m\n\t\tsamples are MSB first. Default is LSB\n"); + fprintf(stderr, "\t-n bits\n\t\tbits per sample\n"); + fprintf(stderr, "\t-p\n\t\tpad RSI to byte boundary\n"); + fprintf(stderr, "\t-r blocks\n\t\treference sample interval in blocks\n"); + fprintf(stderr, "\t-s\n\t\tsamples are signed. Default is unsigned\n"); + fprintf(stderr, "\t-t\n\t\tuse restricted set of code options\n\n"); + return 1; +} diff --git a/src/bdec.sh b/src/bdec.sh new file mode 100755 index 0000000..5b473d8 --- /dev/null +++ b/src/bdec.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e +if [ ! -f bench.rz ]; then + echo "No encoded file found. Encoding now..." + ${top_srcdir}/src/benc.sh ${top_srcdir}/data/typical.rz +fi +rm -f dec.dat +bsize=$(wc -c bench.dat | awk '{print $1}') +utime=$(./utime ./aec -d -n16 -j64 -r256 -m bench.rz dec.dat 2>&1) +perf=$(awk "BEGIN {print ${bsize}/1048576/${utime}}") +echo "*** Decoding with $perf MiB/s user time ***" +cmp bench.dat dec.dat +rm -f dec.dat diff --git a/src/benc.sh b/src/benc.sh new file mode 100755 index 0000000..5bc031c --- /dev/null +++ b/src/benc.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e +TEST_DATA=$1 +AEC=./aec +if [ ! -f bench.dat ]; then + rm -f typical.dat + $AEC -d -n16 -j64 -r256 -m $TEST_DATA typical.dat + for i in $(seq 0 499); + do + cat typical.dat >> bench.dat + done + rm -f typical.dat +fi +rm -f bench.rz +utime=$(./utime $AEC -n16 -j64 -r256 -m bench.dat bench.rz 2>&1) +echo $utime +bsize=$(wc -c bench.dat | awk '{print $1}') +perf=$(awk "BEGIN {print ${bsize}/1048576/${utime}}") +echo "*** Encoding with $perf MiB/s user time ***" diff --git a/src/decode.c b/src/decode.c new file mode 100644 index 0000000..160d4c3 --- /dev/null +++ b/src/decode.c @@ -0,0 +1,842 @@ +/** + * @file decode.c + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Adaptive Entropy Decoder + * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 + * + */ + +#include +#include +#include + +#include "decode.h" +#include "libaec.h" + +#if HAVE_BSR64 +# include +#endif + +#define ROS 5 +#define RSI_USED_SIZE(state) ((size_t)(state->rsip - state->rsi_buffer)) +#define BUFFERSPACE(strm) (strm->avail_in >= strm->state->in_blklen \ + && strm->avail_out >= strm->state->out_blklen) + +#define FLUSH(KIND) \ + static void flush_##KIND(struct aec_stream *strm) \ + { \ + uint32_t *flush_end, *bp, half_d; \ + int32_t data, m; \ + struct internal_state *state = strm->state; \ + \ + flush_end = state->rsip; \ + if (state->pp) { \ + if (state->flush_start == state->rsi_buffer \ + && state->rsip > state->rsi_buffer) { \ + state->last_out = *state->rsi_buffer; \ + \ + if (strm->flags & AEC_DATA_SIGNED) { \ + m = UINT32_C(1) << (strm->bits_per_sample - 1); \ + /* Reference samples have to be sign extended */ \ + state->last_out = (state->last_out ^ m) - m; \ + } \ + put_##KIND(strm, (uint32_t)state->last_out); \ + state->flush_start++; \ + } \ + \ + data = state->last_out; \ + \ + if (state->xmin == 0) { \ + uint32_t xmax, med, d; \ + med = state->xmax / 2 + 1; \ + xmax = state->xmax; \ + \ + for (bp = state->flush_start; bp < flush_end; bp++) { \ + uint32_t mask; \ + d = *bp; \ + half_d = (d >> 1) + (d & 1); \ + /*in this case: data >= med == data & med */ \ + mask = (data & med)?xmax:0; \ + \ + /*in this case: xmax - data == xmax ^ data */ \ + if (half_d <= (mask ^ (uint32_t)data)) { \ + data += (d >> 1)^(~((d & 1) - 1)); \ + } else { \ + data = mask ^ d; \ + } \ + put_##KIND(strm, (uint32_t)data); \ + } \ + state->last_out = data; \ + } else { \ + int32_t xmax, d; \ + xmax = state->xmax; \ + \ + for (bp = state->flush_start; bp < flush_end; bp++) { \ + d = *bp; \ + half_d = ((uint32_t)d >> 1) + (d & 1); \ + \ + if (data < 0) { \ + if (half_d <= xmax + (uint32_t)data + 1) { \ + data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \ + } else { \ + data = d - xmax - 1; \ + } \ + } else { \ + if (half_d <= xmax - (uint32_t)data) { \ + data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \ + } else { \ + data = xmax - d; \ + } \ + } \ + put_##KIND(strm, (uint32_t)data); \ + } \ + state->last_out = data; \ + } \ + } else { \ + for (bp = state->flush_start; bp < flush_end; bp++) \ + put_##KIND(strm, *bp); \ + } \ + state->flush_start = state->rsip; \ + } + + +static inline void put_msb_32(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)(data >> 24); + *strm->next_out++ = (unsigned char)(data >> 16); + *strm->next_out++ = (unsigned char)(data >> 8); + *strm->next_out++ = (unsigned char)data; +} + +static inline void put_msb_24(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)(data >> 16); + *strm->next_out++ = (unsigned char)(data >> 8); + *strm->next_out++ = (unsigned char)data; +} + +static inline void put_msb_16(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)(data >> 8); + *strm->next_out++ = (unsigned char)data; +} + +static inline void put_lsb_32(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)data; + *strm->next_out++ = (unsigned char)(data >> 8); + *strm->next_out++ = (unsigned char)(data >> 16); + *strm->next_out++ = (unsigned char)(data >> 24); +} + +static inline void put_lsb_24(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)data; + *strm->next_out++ = (unsigned char)(data >> 8); + *strm->next_out++ = (unsigned char)(data >> 16); +} + +static inline void put_lsb_16(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)data; + *strm->next_out++ = (unsigned char)(data >> 8); +} + +static inline void put_8(struct aec_stream *strm, uint32_t data) +{ + *strm->next_out++ = (unsigned char)data; +} + +FLUSH(msb_32) +FLUSH(msb_24) +FLUSH(msb_16) +FLUSH(lsb_32) +FLUSH(lsb_24) +FLUSH(lsb_16) +FLUSH(8) + +static inline void put_sample(struct aec_stream *strm, uint32_t s) +{ + struct internal_state *state = strm->state; + + *state->rsip++ = s; + strm->avail_out -= state->bytes_per_sample; +} + +static inline uint32_t direct_get(struct aec_stream *strm, int n) +{ + /** + Get n bit from input stream + + No checking whatsoever. Read bits are dumped. + */ + + struct internal_state *state = strm->state; + if (state->bitp < n) + { + int b = (63 - state->bitp) >> 3; + if (b == 6) { + state->acc = (state->acc << 48) + | ((uint64_t)strm->next_in[0] << 40) + | ((uint64_t)strm->next_in[1] << 32) + | ((uint64_t)strm->next_in[2] << 24) + | ((uint64_t)strm->next_in[3] << 16) + | ((uint64_t)strm->next_in[4] << 8) + | (uint64_t)strm->next_in[5]; + } else if (b == 7) { + state->acc = (state->acc << 56) + | ((uint64_t)strm->next_in[0] << 48) + | ((uint64_t)strm->next_in[1] << 40) + | ((uint64_t)strm->next_in[2] << 32) + | ((uint64_t)strm->next_in[3] << 24) + | ((uint64_t)strm->next_in[4] << 16) + | ((uint64_t)strm->next_in[5] << 8) + | (uint64_t)strm->next_in[6]; + } else if (b == 5) { + state->acc = (state->acc << 40) + | ((uint64_t)strm->next_in[0] << 32) + | ((uint64_t)strm->next_in[1] << 24) + | ((uint64_t)strm->next_in[2] << 16) + | ((uint64_t)strm->next_in[3] << 8) + | (uint64_t)strm->next_in[4]; + } else if (b == 4) { + state->acc = (state->acc << 32) + | ((uint64_t)strm->next_in[0] << 24) + | ((uint64_t)strm->next_in[1] << 16) + | ((uint64_t)strm->next_in[2] << 8) + | (uint64_t)strm->next_in[3]; + } else if (b == 3) { + state->acc = (state->acc << 24) + | ((uint64_t)strm->next_in[0] << 16) + | ((uint64_t)strm->next_in[1] << 8) + | (uint64_t)strm->next_in[2]; + } else if (b == 2) { + state->acc = (state->acc << 16) + | ((uint64_t)strm->next_in[0] << 8) + | (uint64_t)strm->next_in[1]; + } else if (b == 1) { + state->acc = (state->acc << 8) + | (uint64_t)strm->next_in[0]; + } + strm->next_in += b; + strm->avail_in -= b; + state->bitp += b << 3; + } + + state->bitp -= n; + return (state->acc >> state->bitp) & (UINT64_MAX >> (64 - n)); +} + +static inline uint32_t direct_get_fs(struct aec_stream *strm) +{ + /** + Interpret a Fundamental Sequence from the input buffer. + + Essentially counts the number of 0 bits until a 1 is + encountered. + */ + + uint32_t fs = 0; + struct internal_state *state = strm->state; + + if (state->bitp) + state->acc &= UINT64_MAX >> (64 - state->bitp); + else + state->acc = 0; + + while (state->acc == 0) { + if (strm->avail_in < 7) + return 0; + + state->acc = (state->acc << 56) + | ((uint64_t)strm->next_in[0] << 48) + | ((uint64_t)strm->next_in[1] << 40) + | ((uint64_t)strm->next_in[2] << 32) + | ((uint64_t)strm->next_in[3] << 24) + | ((uint64_t)strm->next_in[4] << 16) + | ((uint64_t)strm->next_in[5] << 8) + | (uint64_t)strm->next_in[6]; + strm->next_in += 7; + strm->avail_in -= 7; + fs += state->bitp; + state->bitp = 56; + } + + { +#ifndef __has_builtin +#define __has_builtin(x) 0 /* Compatibility with non-clang compilers. */ +#endif +#if HAVE_DECL___BUILTIN_CLZLL || __has_builtin(__builtin_clzll) + int i = 63 - __builtin_clzll(state->acc); +#elif HAVE_BSR64 + unsigned long i; + _BitScanReverse64(&i, state->acc); +#else + int i = state->bitp - 1; + while ((state->acc & (UINT64_C(1) << i)) == 0) + i--; +#endif + fs += state->bitp - i - 1; + state->bitp = i; + } + return fs; +} + +static inline uint32_t bits_ask(struct aec_stream *strm, int n) +{ + while (strm->state->bitp < n) { + if (strm->avail_in == 0) + return 0; + strm->avail_in--; + strm->state->acc <<= 8; + strm->state->acc |= *strm->next_in++; + strm->state->bitp += 8; + } + return 1; +} + +static inline uint32_t bits_get(struct aec_stream *strm, int n) +{ + return (strm->state->acc >> (strm->state->bitp - n)) + & (UINT64_MAX >> (64 - n)); +} + +static inline void bits_drop(struct aec_stream *strm, int n) +{ + strm->state->bitp -= n; +} + +static inline uint32_t fs_ask(struct aec_stream *strm) +{ + if (bits_ask(strm, 1) == 0) + return 0; + while ((strm->state->acc & (UINT64_C(1) << (strm->state->bitp - 1))) == 0) { + if (strm->state->bitp == 1) { + if (strm->avail_in == 0) + return 0; + strm->avail_in--; + strm->state->acc <<= 8; + strm->state->acc |= *strm->next_in++; + strm->state->bitp += 8; + } + strm->state->fs++; + strm->state->bitp--; + } + return 1; +} + +static inline void fs_drop(struct aec_stream *strm) +{ + strm->state->fs = 0; + strm->state->bitp--; +} + +static inline uint32_t copysample(struct aec_stream *strm) +{ + if (bits_ask(strm, strm->bits_per_sample) == 0 + || strm->avail_out < strm->state->bytes_per_sample) + return 0; + + put_sample(strm, bits_get(strm, strm->bits_per_sample)); + bits_drop(strm, strm->bits_per_sample); + return 1; +} + +static inline int m_id(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + if (strm->avail_in >= strm->state->in_blklen) { + state->id = direct_get(strm, state->id_len); + } else { + if (bits_ask(strm, state->id_len) == 0) { + state->mode = m_id; + return M_EXIT; + } + state->id = bits_get(strm, state->id_len); + bits_drop(strm, state->id_len); + } + state->mode = state->id_table[state->id]; + return(state->mode(strm)); +} + +static int m_next_cds(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + if (state->rsi_size == RSI_USED_SIZE(state)) { + state->flush_output(strm); + state->flush_start = state->rsi_buffer; + state->rsip = state->rsi_buffer; + if (state->pp) { + state->ref = 1; + state->encoded_block_size = strm->block_size - 1; + } + if (strm->flags & AEC_PAD_RSI) + state->bitp -= state->bitp % 8; + } else { + state->ref = 0; + state->encoded_block_size = strm->block_size; + } + return m_id(strm); +} + +static int m_split_output(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + int k = state->id - 1; + + do { + if (bits_ask(strm, k) == 0 || strm->avail_out < state->bytes_per_sample) + return M_EXIT; + if (k) + *state->rsip++ += bits_get(strm, k); + else + state->rsip++; + strm->avail_out -= state->bytes_per_sample; + bits_drop(strm, k); + } while(++state->sample_counter < state->encoded_block_size); + + state->mode = m_next_cds; + return M_CONTINUE; +} + +static int m_split_fs(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + int k = state->id - 1; + + do { + if (fs_ask(strm) == 0) + return M_EXIT; + state->rsip[state->sample_counter] = state->fs << k; + fs_drop(strm); + } while(++state->sample_counter < state->encoded_block_size); + + state->sample_counter = 0; + state->mode = m_split_output; + + return M_CONTINUE; +} + +static int m_split(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (BUFFERSPACE(strm)) { + int k = state->id - 1; + size_t binary_part = (k * state->encoded_block_size) / 8 + 9; + + if (state->ref) + *state->rsip++ = direct_get(strm, strm->bits_per_sample); + + for (size_t i = 0; i < state->encoded_block_size; i++) + state->rsip[i] = direct_get_fs(strm) << k; + + if (k) { + if (strm->avail_in < binary_part) + return M_ERROR; + + for (size_t i = 0; i < state->encoded_block_size; i++) + *state->rsip++ += direct_get(strm, k); + } else { + state->rsip += state->encoded_block_size; + } + + strm->avail_out -= state->out_blklen; + state->mode = m_next_cds; + } else { + if (state->ref && (copysample(strm) == 0)) + return M_EXIT; + state->sample_counter = 0; + state->mode = m_split_fs; + } + return M_CONTINUE; +} + +static int m_zero_output(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + do { + if (strm->avail_out < state->bytes_per_sample) + return M_EXIT; + put_sample(strm, 0); + } while(--state->sample_counter); + + state->mode = m_next_cds; + return M_CONTINUE; +} + +static int m_zero_block(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + uint32_t zero_blocks; + uint32_t zero_samples; + uint32_t zero_bytes; + + if (fs_ask(strm) == 0) + return M_EXIT; + + zero_blocks = state->fs + 1; + fs_drop(strm); + + if (zero_blocks == ROS) { + int b = (int)RSI_USED_SIZE(state) / strm->block_size; + zero_blocks = MIN((int)(strm->rsi - b), 64 - (b % 64)); + } else if (zero_blocks > ROS) { + zero_blocks--; + } + + zero_samples = zero_blocks * strm->block_size - state->ref; + if (state->rsi_size - RSI_USED_SIZE(state) < zero_samples) + return M_ERROR; + + zero_bytes = zero_samples * state->bytes_per_sample; + if (strm->avail_out >= zero_bytes) { + memset(state->rsip, 0, zero_samples * sizeof(uint32_t)); + state->rsip += zero_samples; + strm->avail_out -= zero_bytes; + state->mode = m_next_cds; + } else { + state->sample_counter = zero_samples; + state->mode = m_zero_output; + } + return M_CONTINUE; +} + +static int m_se_decode(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + while(state->sample_counter < strm->block_size) { + int32_t m; + int32_t d1; + if (fs_ask(strm) == 0) + return M_EXIT; + m = state->fs; + if (m > SE_TABLE_SIZE) + return M_ERROR; + d1 = m - state->se_table[2 * m + 1]; + + if ((state->sample_counter & 1) == 0) { + if (strm->avail_out < state->bytes_per_sample) + return M_EXIT; + put_sample(strm, state->se_table[2 * m] - d1); + state->sample_counter++; + } + + if (strm->avail_out < state->bytes_per_sample) + return M_EXIT; + put_sample(strm, d1); + state->sample_counter++; + fs_drop(strm); + } + + state->mode = m_next_cds; + return M_CONTINUE; +} + +static int m_se(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (BUFFERSPACE(strm)) { + uint32_t i = state->ref; + + while (i < strm->block_size) { + uint32_t m = direct_get_fs(strm); + int32_t d1; + + if (m > SE_TABLE_SIZE) + return M_ERROR; + + d1 = m - state->se_table[2 * m + 1]; + + if ((i & 1) == 0) { + put_sample(strm, state->se_table[2 * m] - d1); + i++; + } + put_sample(strm, d1); + i++; + } + state->mode = m_next_cds; + } else { + state->mode = m_se_decode; + state->sample_counter = state->ref; + } + return M_CONTINUE; +} + +static int m_low_entropy_ref(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (state->ref && copysample(strm) == 0) + return M_EXIT; + + if (state->id == 1) + state->mode = m_se; + else + state->mode = m_zero_block; + return M_CONTINUE; +} + +static int m_low_entropy(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (bits_ask(strm, 1) == 0) + return M_EXIT; + state->id = bits_get(strm, 1); + bits_drop(strm, 1); + state->mode = m_low_entropy_ref; + return M_CONTINUE; +} + +static int m_uncomp_copy(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + do { + if (copysample(strm) == 0) + return M_EXIT; + } while(--state->sample_counter); + + state->mode = m_next_cds; + return M_CONTINUE; +} + +static int m_uncomp(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (BUFFERSPACE(strm)) { + for (size_t i = 0; i < strm->block_size; i++) + *state->rsip++ = direct_get(strm, strm->bits_per_sample); + strm->avail_out -= state->out_blklen; + state->mode = m_next_cds; + } else { + state->sample_counter = strm->block_size; + state->mode = m_uncomp_copy; + } + return M_CONTINUE; +} + +static void create_se_table(int *table) +{ + int k = 0; + for (int i = 0; i < 13; i++) { + int ms = k; + for (int j = 0; j <= i; j++) { + table[2 * k] = i; + table[2 * k + 1] = ms; + k++; + } + } +} + +int aec_decode_init(struct aec_stream *strm) +{ + struct internal_state *state; + int modi; + + if (strm->bits_per_sample > 32 || strm->bits_per_sample == 0) + return AEC_CONF_ERROR; + + state = malloc(sizeof(struct internal_state)); + if (state == NULL) + return AEC_MEM_ERROR; + memset(state, 0, sizeof(struct internal_state)); + + create_se_table(state->se_table); + + strm->state = state; + + if (strm->bits_per_sample > 16) { + state->id_len = 5; + + if (strm->bits_per_sample <= 24 && strm->flags & AEC_DATA_3BYTE) { + state->bytes_per_sample = 3; + if (strm->flags & AEC_DATA_MSB) + state->flush_output = flush_msb_24; + else + state->flush_output = flush_lsb_24; + } else { + state->bytes_per_sample = 4; + if (strm->flags & AEC_DATA_MSB) + state->flush_output = flush_msb_32; + else + state->flush_output = flush_lsb_32; + } + state->out_blklen = strm->block_size * state->bytes_per_sample; + } + else if (strm->bits_per_sample > 8) { + state->bytes_per_sample = 2; + state->id_len = 4; + state->out_blklen = strm->block_size * 2; + if (strm->flags & AEC_DATA_MSB) + state->flush_output = flush_msb_16; + else + state->flush_output = flush_lsb_16; + } else { + if (strm->flags & AEC_RESTRICTED) { + if (strm->bits_per_sample <= 4) { + if (strm->bits_per_sample <= 2) + state->id_len = 1; + else + state->id_len = 2; + } else { + return AEC_CONF_ERROR; + } + } else { + state->id_len = 3; + } + + state->bytes_per_sample = 1; + state->out_blklen = strm->block_size; + state->flush_output = flush_8; + } + + if (strm->flags & AEC_DATA_SIGNED) { + state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); + state->xmin = ~state->xmax; + } else { + state->xmin = 0; + state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample); + } + + state->in_blklen = (strm->block_size * strm->bits_per_sample + + state->id_len) / 8 + 16; + + modi = 1UL << state->id_len; + state->id_table = malloc(modi * sizeof(int (*)(struct aec_stream *))); + if (state->id_table == NULL) + return AEC_MEM_ERROR; + + state->id_table[0] = m_low_entropy; + for (int i = 1; i < modi - 1; i++) { + state->id_table[i] = m_split; + } + state->id_table[modi - 1] = m_uncomp; + + state->rsi_size = strm->rsi * strm->block_size; + state->rsi_buffer = malloc(state->rsi_size * sizeof(uint32_t)); + if (state->rsi_buffer == NULL) + return AEC_MEM_ERROR; + + state->pp = strm->flags & AEC_DATA_PREPROCESS; + if (state->pp) { + state->ref = 1; + state->encoded_block_size = strm->block_size - 1; + } else { + state->ref = 0; + state->encoded_block_size = strm->block_size; + } + strm->total_in = 0; + strm->total_out = 0; + + state->rsip = state->rsi_buffer; + state->flush_start = state->rsi_buffer; + state->bitp = 0; + state->fs = 0; + state->mode = m_id; + return AEC_OK; +} + +int aec_decode(struct aec_stream *strm, int flush) +{ + /** + Finite-state machine implementation of the adaptive entropy + decoder. + + Can work with one byte input und one sample output buffers. If + enough buffer space is available, then faster implementations + of the states are called. Inspired by zlib. + */ + + struct internal_state *state = strm->state; + int status; + + strm->total_in += strm->avail_in; + strm->total_out += strm->avail_out; + + do { + status = state->mode(strm); + } while (status == M_CONTINUE); + + if (status == M_ERROR) + return AEC_DATA_ERROR; + + if (status == M_EXIT && strm->avail_out > 0 && + strm->avail_out < state->bytes_per_sample) + return AEC_MEM_ERROR; + + state->flush_output(strm); + + strm->total_in -= strm->avail_in; + strm->total_out -= strm->avail_out; + + return AEC_OK; +} + +int aec_decode_end(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + free(state->id_table); + free(state->rsi_buffer); + free(state); + return AEC_OK; +} + +int aec_buffer_decode(struct aec_stream *strm) +{ + int status = aec_decode_init(strm); + if (status != AEC_OK) + return status; + + status = aec_decode(strm, AEC_FLUSH); + aec_decode_end(strm); + return status; +} diff --git a/src/decode.h b/src/decode.h new file mode 100644 index 0000000..78cdbcb --- /dev/null +++ b/src/decode.h @@ -0,0 +1,139 @@ +/** + * @file decode.c + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Adaptive Entropy Decoder + * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 + * + */ + +#ifndef DECODE_H +#define DECODE_H 1 + +#include + +#if HAVE_STDINT_H +# include +#endif + +#define M_CONTINUE 1 +#define M_EXIT 0 +#define M_ERROR (-1) + +#define MIN(a, b) (((a) < (b))? (a): (b)) + +#define SE_TABLE_SIZE 90 + +struct aec_stream; + +struct internal_state { + int (*mode)(struct aec_stream *); + + /* option ID */ + int id; + + /* bit length of code option identification key */ + int id_len; + + /* table maps IDs to states */ + int (**id_table)(struct aec_stream *); + + void (*flush_output)(struct aec_stream *); + + /* previous output for post-processing */ + int32_t last_out; + + /* minimum integer for post-processing */ + uint32_t xmin; + + /* maximum integer for post-processing */ + uint32_t xmax; + + /* length of uncompressed input block should be the longest + legal block */ + uint32_t in_blklen; + + /* length of output block in bytes */ + uint32_t out_blklen; + + uint32_t sample_counter; + + /* accumulator for currently used bit sequence */ + uint64_t acc; + + /* bit pointer to the next unused bit in accumulator */ + int bitp; + + /* last fundamental sequence in accumulator */ + int fs; + + /* 1 if current block has reference sample */ + int ref; + + /* block_size minus reference sample if present */ + uint32_t encoded_block_size; + + /* 1 if postprocessor has to be used */ + int pp; + + /* storage size of samples in bytes */ + uint32_t bytes_per_sample; + + /* output buffer holding one reference sample interval */ + uint32_t *rsi_buffer; + + /* current position of output in rsi_buffer */ + uint32_t *rsip; + + /* rsi in bytes */ + size_t rsi_size; + + /* first not yet flushed byte in rsi_buffer */ + uint32_t *flush_start; + + /* table for decoding second extension option */ + int se_table[2 * (SE_TABLE_SIZE + 1)]; +} decode_state; + +#endif /* DECODE_H */ diff --git a/src/encode.c b/src/encode.c new file mode 100644 index 0000000..cbaf8af --- /dev/null +++ b/src/encode.c @@ -0,0 +1,948 @@ +/** + * @file encode.c + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Adaptive Entropy Encoder + * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 + * + */ + +#include +#include +#include + +#include "encode.h" +#include "encode_accessors.h" +#include "libaec.h" + +static int m_get_block(struct aec_stream *strm); + +static inline void emit(struct internal_state *state, + uint32_t data, int bits) +{ + /** + Emit sequence of bits. + */ + + if (bits <= state->bits) { + state->bits -= bits; + *state->cds += (uint8_t)(data << state->bits); + } else { + bits -= state->bits; + *state->cds++ += (uint8_t)((uint64_t)data >> bits); + + while (bits > 8) { + bits -= 8; + *state->cds++ = (uint8_t)(data >> bits); + } + + state->bits = 8 - bits; + *state->cds = (uint8_t)(data << state->bits); + } +} + +static inline void emitfs(struct internal_state *state, int fs) +{ + /** + Emits a fundamental sequence. + + fs zero bits followed by one 1 bit. + */ + + for(;;) { + if (fs < state->bits) { + state->bits -= fs + 1; + *state->cds += 1U << state->bits; + break; + } else { + fs -= state->bits; + *++state->cds = 0; + state->bits = 8; + } + } +} + +static inline void copy64(uint8_t *dst, uint64_t src) +{ + dst[0] = (uint8_t)(src >> 56); + dst[1] = (uint8_t)(src >> 48); + dst[2] = (uint8_t)(src >> 40); + dst[3] = (uint8_t)(src >> 32); + dst[4] = (uint8_t)(src >> 24); + dst[5] = (uint8_t)(src >> 16); + dst[6] = (uint8_t)(src >> 8); + dst[7] = (uint8_t)src; +} + +static inline void emitblock_fs(struct aec_stream *strm, int k, int ref) +{ + struct internal_state *state = strm->state; + + uint64_t acc = (uint64_t)*state->cds << 56; + uint32_t used = 7 - state->bits; /* used bits in 64 bit accumulator */ + + for (size_t i = ref; i < strm->block_size; i++) { + used += (state->block[i] >> k) + 1; + while (used > 63) { + copy64(state->cds, acc); + state->cds += 8; + acc = 0; + used -= 64; + } + acc |= UINT64_C(1) << (63 - used); + } + + copy64(state->cds, acc); + state->cds += used >> 3; + state->bits = 7 - (used & 7); +} + +static inline void emitblock(struct aec_stream *strm, int k, int ref) +{ + /** + Emit the k LSB of a whole block of input data. + */ + + struct internal_state *state = strm->state; + uint32_t *in = state->block + ref; + uint32_t *in_end = state->block + strm->block_size; + uint64_t mask = (UINT64_C(1) << k) - 1; + uint8_t *o = state->cds; + uint64_t a = *o; + int p = state->bits; + + while(in < in_end) { + a <<= 56; + p = (p % 8) + 56; + + while (p > k && in < in_end) { + p -= k; + a += ((uint64_t)(*in++) & mask) << p; + } + + switch (p & ~7) { + case 0: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + o[2] = (uint8_t)(a >> 40); + o[3] = (uint8_t)(a >> 32); + o[4] = (uint8_t)(a >> 24); + o[5] = (uint8_t)(a >> 16); + o[6] = (uint8_t)(a >> 8); + o += 7; + break; + case 8: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + o[2] = (uint8_t)(a >> 40); + o[3] = (uint8_t)(a >> 32); + o[4] = (uint8_t)(a >> 24); + o[5] = (uint8_t)(a >> 16); + a >>= 8; + o += 6; + break; + case 16: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + o[2] = (uint8_t)(a >> 40); + o[3] = (uint8_t)(a >> 32); + o[4] = (uint8_t)(a >> 24); + a >>= 16; + o += 5; + break; + case 24: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + o[2] = (uint8_t)(a >> 40); + o[3] = (uint8_t)(a >> 32); + a >>= 24; + o += 4; + break; + case 32: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + o[2] = (uint8_t)(a >> 40); + a >>= 32; + o += 3; + break; + case 40: + o[0] = (uint8_t)(a >> 56); + o[1] = (uint8_t)(a >> 48); + a >>= 40; + o += 2; + break; + case 48: + *o++ = (uint8_t)(a >> 56); + a >>= 48; + break; + default: + a >>= 56; + break; + } + } + + *o = (uint8_t)a; + state->cds = o; + state->bits = p % 8; +} + +static void preprocess_unsigned(struct aec_stream *strm) +{ + /** + Preprocess RSI of unsigned samples. + + Combining preprocessing and converting to uint32_t in one loop + is slower due to the data dependance on x_i-1. + */ + + uint32_t D; + struct internal_state *state = strm->state; + const uint32_t *restrict x = state->data_raw; + uint32_t *restrict d = state->data_pp; + uint32_t xmax = state->xmax; + uint32_t rsi = strm->rsi * strm->block_size - 1; + + state->ref = 1; + state->ref_sample = x[0]; + d[0] = 0; + for (size_t i = 0; i < rsi; i++) { + if (x[i + 1] >= x[i]) { + D = x[i + 1] - x[i]; + if (D <= x[i]) + d[i + 1] = 2 * D; + else + d[i + 1] = x[i + 1]; + } else { + D = x[i] - x[i + 1]; + if (D <= xmax - x[i]) + d[i + 1] = 2 * D - 1; + else + d[i + 1] = xmax - x[i + 1]; + } + } + state->uncomp_len = (strm->block_size - 1) * strm->bits_per_sample; +} + +static void preprocess_signed(struct aec_stream *strm) +{ + /** + Preprocess RSI of signed samples. + */ + + uint32_t D; + struct internal_state *state = strm->state; + int32_t *restrict x = (int32_t *)state->data_raw; + uint32_t *restrict d = state->data_pp; + int32_t xmax = (int32_t)state->xmax; + int32_t xmin = (int32_t)state->xmin; + uint32_t rsi = strm->rsi * strm->block_size - 1; + uint32_t m = UINT64_C(1) << (strm->bits_per_sample - 1); + + state->ref = 1; + state->ref_sample = x[0]; + d[0] = 0; + x[0] = (x[0] ^ m) - m; + + for (size_t i = 0; i < rsi; i++) { + x[i + 1] = (x[i + 1] ^ m) - m; + if (x[i + 1] < x[i]) { + D = (uint32_t)(x[i] - x[i + 1]); + if (D <= (uint32_t)(xmax - x[i])) + d[i + 1] = 2 * D - 1; + else + d[i + 1] = xmax - x[i + 1]; + } else { + D = (uint32_t)(x[i + 1] - x[i]); + if (D <= (uint32_t)(x[i] - xmin)) + d[i + 1] = 2 * D; + else + d[i + 1] = x[i + 1] - xmin; + } + } + state->uncomp_len = (strm->block_size - 1) * strm->bits_per_sample; +} + +static inline uint64_t block_fs(struct aec_stream *strm, int k) +{ + /** + Sum FS of all samples in block for given splitting position. + */ + + struct internal_state *state = strm->state; + uint64_t fs = 0; + + for (size_t i = 0; i < strm->block_size; i++) + fs += (uint64_t)(state->block[i] >> k); + + return fs; +} + +static uint32_t assess_splitting_option(struct aec_stream *strm) +{ + /** + Length of CDS encoded with splitting option and optimal k. + + In Rice coding each sample in a block of samples is split at + the same position into k LSB and bits_per_sample - k MSB. The + LSB part is left binary and the MSB part is coded as a + fundamental sequence a.k.a. unary (see CCSDS 121.0-B-2). The + function of the length of the Coded Data Set (CDS) depending on + k has exactly one minimum (see A. Kiely, IPN Progress Report + 42-159). + + To find that minimum with only a few costly evaluations of the + CDS length, we start with the k of the previous CDS. K is + increased and the CDS length evaluated. If the CDS length gets + smaller, then we are moving towards the minimum. If the length + increases, then the minimum will be found with smaller k. + + For increasing k we know that we will gain block_size bits in + length through the larger binary part. If the FS lenth is less + than the block size then a reduced FS part can't compensate the + larger binary part. So we know that the CDS for k+1 will be + larger than for k without actually computing the length. An + analogue check can be done for decreasing k. + */ + + struct internal_state *state = strm->state; + + /* Block size of current block */ + int this_bs = strm->block_size - state->ref; + + /* CDS length minimum so far */ + uint64_t len_min = UINT64_MAX; + + int k = state->k; + int k_min = k; + + /* 1 if we shouldn't reverse */ + int no_turn = (k == 0); + + /* Direction, 1 means increasing k, 0 decreasing k */ + int dir = 1; + + for (;;) { + /* Length of FS part (not including 1s) */ + uint64_t fs_len = block_fs(strm, k); + + /* CDS length for current k */ + uint64_t len = fs_len + this_bs * (k + 1); + + if (len < len_min) { + if (len_min < UINT64_MAX) + no_turn = 1; + + len_min = len; + k_min = k; + + if (dir) { + if (fs_len < this_bs || k >= state->kmax) { + if (no_turn) + break; + k = state->k - 1; + dir = 0; + no_turn = 1; + } else { + k++; + } + } else { + if (fs_len >= this_bs || k == 0) + break; + k--; + } + } else { + if (no_turn) + break; + k = state->k - 1; + dir = 0; + no_turn = 1; + } + } + state->k = k_min; + + return (uint32_t)len_min; +} + +static uint32_t assess_se_option(struct aec_stream *strm) +{ + /** + Length of CDS encoded with Second Extension option. + + If length is above limit just return UINT32_MAX. + */ + + struct internal_state *state = strm->state; + uint32_t *block = state->block; + uint64_t len = 1; + + for (size_t i = 0; i < strm->block_size; i += 2) { + uint64_t d = (uint64_t)block[i] + (uint64_t)block[i + 1]; + len += d * (d + 1) / 2 + block[i + 1] + 1; + if (len > state->uncomp_len) + return UINT32_MAX; + } + return (uint32_t)len; +} + +static void init_output(struct aec_stream *strm) +{ + /** + Direct output to next_out if next_out can hold a Coded Data + Set, use internal buffer otherwise. + */ + + struct internal_state *state = strm->state; + + if (strm->avail_out > CDSLEN) { + if (!state->direct_out) { + state->direct_out = 1; + *strm->next_out = *state->cds; + state->cds = strm->next_out; + } + } else { + if (state->zero_blocks == 0 || state->direct_out) { + /* copy leftover from last block */ + *state->cds_buf = *state->cds; + state->cds = state->cds_buf; + } + state->direct_out = 0; + } +} + +/* + * + * FSM functions + * + */ + +static int m_flush_block_resumable(struct aec_stream *strm) +{ + /** + Slow and restartable flushing + */ + struct internal_state *state = strm->state; + + int n = (int)MIN((size_t)(state->cds - state->cds_buf - state->i), + strm->avail_out); + memcpy(strm->next_out, state->cds_buf + state->i, n); + strm->next_out += n; + strm->avail_out -= n; + state->i += n; + + if (strm->avail_out == 0) { + return M_EXIT; + } else { + state->mode = m_get_block; + return M_CONTINUE; + } +} + +static int m_flush_block(struct aec_stream *strm) +{ + /** + Flush block in direct_out mode by updating counters. + + Fall back to slow flushing if in buffered mode. + */ + struct internal_state *state = strm->state; + +#ifdef ENABLE_RSI_PADDING + if (state->blocks_avail == 0 + && strm->flags & AEC_PAD_RSI + && state->block_nonzero == 0 + ) + emit(state, 0, state->bits % 8); +#endif + + if (state->direct_out) { + int n = (int)(state->cds - strm->next_out); + strm->next_out += n; + strm->avail_out -= n; + state->mode = m_get_block; + return M_CONTINUE; + } + + state->i = 0; + state->mode = m_flush_block_resumable; + return M_CONTINUE; +} + +static int m_encode_splitting(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + int k = state->k; + + emit(state, k + 1, state->id_len); + if (state->ref) + emit(state, state->ref_sample, strm->bits_per_sample); + + emitblock_fs(strm, k, state->ref); + if (k) + emitblock(strm, k, state->ref); + + return m_flush_block(strm); +} + +static int m_encode_uncomp(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + emit(state, (1U << state->id_len) - 1, state->id_len); + if (state->ref) + state->block[0] = state->ref_sample; + emitblock(strm, strm->bits_per_sample, 0); + return m_flush_block(strm); +} + +static int m_encode_se(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + emit(state, 1, state->id_len + 1); + if (state->ref) + emit(state, state->ref_sample, strm->bits_per_sample); + + for (size_t i = 0; i < strm->block_size; i+= 2) { + uint32_t d = state->block[i] + state->block[i + 1]; + emitfs(state, d * (d + 1) / 2 + state->block[i + 1]); + } + + return m_flush_block(strm); +} + +static int m_encode_zero(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + emit(state, 0, state->id_len + 1); + + if (state->zero_ref) + emit(state, state->zero_ref_sample, strm->bits_per_sample); + + if (state->zero_blocks == ROS) + emitfs(state, 4); + else if (state->zero_blocks >= 5) + emitfs(state, state->zero_blocks); + else + emitfs(state, state->zero_blocks - 1); + + state->zero_blocks = 0; + return m_flush_block(strm); +} + +static int m_select_code_option(struct aec_stream *strm) +{ + /** + Decide which code option to use. + */ + + struct internal_state *state = strm->state; + + uint32_t split_len; + uint32_t se_len; + if (state->id_len > 1) + split_len = assess_splitting_option(strm); + else + split_len = UINT32_MAX; + se_len = assess_se_option(strm); + + if (split_len < state->uncomp_len) { + if (split_len < se_len) + return m_encode_splitting(strm); + else + return m_encode_se(strm); + } else { + if (state->uncomp_len <= se_len) + return m_encode_uncomp(strm); + else + return m_encode_se(strm); + } +} + +static int m_check_zero_block(struct aec_stream *strm) +{ + /** + Check if input block is all zero. + + Aggregate consecutive zero blocks until we find !0 or reach the + end of a segment or RSI. + */ + + struct internal_state *state = strm->state; + uint32_t *p = state->block; + + size_t i; + for (i = 0; i < strm->block_size; i++) + if (p[i] != 0) + break; + + if (i < strm->block_size) { + if (state->zero_blocks) { + /* The current block isn't zero but we have to emit a + * previous zero block first. The current block will be + * flagged and handled later. + */ + state->block_nonzero = 1; + state->mode = m_encode_zero; + return M_CONTINUE; + } + state->mode = m_select_code_option; + return M_CONTINUE; + } else { + state->zero_blocks++; + if (state->zero_blocks == 1) { + state->zero_ref = state->ref; + state->zero_ref_sample = state->ref_sample; + } + if (state->blocks_avail == 0 || state->blocks_dispensed % 64 == 0) { + if (state->zero_blocks > 4) + state->zero_blocks = ROS; + + state->mode = m_encode_zero; + return M_CONTINUE; + } + state->mode = m_get_block; + return M_CONTINUE; + } +} + +static int m_get_rsi_resumable(struct aec_stream *strm) +{ + /** + Get RSI while input buffer is short. + + Let user provide more input. Once we got all input pad buffer + to full RSI. + */ + + struct internal_state *state = strm->state; + + do { + if (strm->avail_in >= state->bytes_per_sample) { + state->data_raw[state->i] = state->get_sample(strm); + } else { + if (state->flush == AEC_FLUSH) { + if (state->i > 0) { + state->blocks_avail = state->i / strm->block_size - 1; + if (state->i % strm->block_size) + state->blocks_avail++; + do + state->data_raw[state->i] = + state->data_raw[state->i - 1]; + while(++state->i < strm->rsi * strm->block_size); + } else { + /* Finish encoding by padding the last byte with + * zero bits. */ + emit(state, 0, state->bits); + if (strm->avail_out > 0) { + if (!state->direct_out) + *strm->next_out++ = *state->cds; + strm->avail_out--; + state->flushed = 1; + } + return M_EXIT; + } + } else { + return M_EXIT; + } + } + } while (++state->i < strm->rsi * strm->block_size); + + if (strm->flags & AEC_DATA_PREPROCESS) + state->preprocess(strm); + + return m_check_zero_block(strm); +} + +static int m_get_block(struct aec_stream *strm) +{ + /** + Provide the next block of preprocessed input data. + + Pull in a whole Reference Sample Interval (RSI) of data if + block buffer is empty. + */ + + struct internal_state *state = strm->state; + + init_output(strm); + + if (state->block_nonzero) { + state->block_nonzero = 0; + state->mode = m_select_code_option; + return M_CONTINUE; + } + + if (state->blocks_avail == 0) { + state->blocks_avail = strm->rsi - 1; + state->block = state->data_pp; + state->blocks_dispensed = 1; + + if (strm->avail_in >= state->rsi_len) { + state->get_rsi(strm); + if (strm->flags & AEC_DATA_PREPROCESS) + state->preprocess(strm); + + return m_check_zero_block(strm); + } else { + state->i = 0; + state->mode = m_get_rsi_resumable; + } + } else { + if (state->ref) { + state->ref = 0; + state->uncomp_len = strm->block_size * strm->bits_per_sample; + } + state->block += strm->block_size; + state->blocks_dispensed++; + state->blocks_avail--; + return m_check_zero_block(strm); + } + return M_CONTINUE; +} + +static void cleanup(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + if (strm->flags & AEC_DATA_PREPROCESS && state->data_raw) + free(state->data_raw); + if (state->data_pp) + free(state->data_pp); + free(state); +} + +/* + * + * API functions + * + */ + +int aec_encode_init(struct aec_stream *strm) +{ + struct internal_state *state; + + if (strm->bits_per_sample > 32 || strm->bits_per_sample == 0) + return AEC_CONF_ERROR; + + if (strm->flags & AEC_NOT_ENFORCE) { + /* All even block sizes are allowed. */ + if (strm->block_size & 1) + return AEC_CONF_ERROR; + } else { + /* Only allow standard conforming block sizes */ + if (strm->block_size != 8 + && strm->block_size != 16 + && strm->block_size != 32 + && strm->block_size != 64) + return AEC_CONF_ERROR; + } + + if (strm->rsi > 4096) + return AEC_CONF_ERROR; + + state = malloc(sizeof(struct internal_state)); + if (state == NULL) + return AEC_MEM_ERROR; + + memset(state, 0, sizeof(struct internal_state)); + strm->state = state; + state->uncomp_len = strm->block_size * strm->bits_per_sample; + + if (strm->bits_per_sample > 16) { + /* 24/32 input bit settings */ + state->id_len = 5; + + if (strm->bits_per_sample <= 24 + && strm->flags & AEC_DATA_3BYTE) { + state->bytes_per_sample = 3; + if (strm->flags & AEC_DATA_MSB) { + state->get_sample = aec_get_msb_24; + state->get_rsi = aec_get_rsi_msb_24; + } else { + state->get_sample = aec_get_lsb_24; + state->get_rsi = aec_get_rsi_lsb_24; + } + } else { + state->bytes_per_sample = 4; + if (strm->flags & AEC_DATA_MSB) { + state->get_sample = aec_get_msb_32; + state->get_rsi = aec_get_rsi_msb_32; + } else { + state->get_sample = aec_get_lsb_32; + state->get_rsi = aec_get_rsi_lsb_32; + } + } + } + else if (strm->bits_per_sample > 8) { + /* 16 bit settings */ + state->id_len = 4; + state->bytes_per_sample = 2; + + if (strm->flags & AEC_DATA_MSB) { + state->get_sample = aec_get_msb_16; + state->get_rsi = aec_get_rsi_msb_16; + } else { + state->get_sample = aec_get_lsb_16; + state->get_rsi = aec_get_rsi_lsb_16; + } + } else { + /* 8 bit settings */ + if (strm->flags & AEC_RESTRICTED) { + if (strm->bits_per_sample <= 4) { + if (strm->bits_per_sample <= 2) + state->id_len = 1; + else + state->id_len = 2; + } else { + return AEC_CONF_ERROR; + } + } else { + state->id_len = 3; + } + state->bytes_per_sample = 1; + + state->get_sample = aec_get_8; + state->get_rsi = aec_get_rsi_8; + } + state->rsi_len = strm->rsi * strm->block_size * state->bytes_per_sample; + + if (strm->flags & AEC_DATA_SIGNED) { + state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); + state->xmin = ~state->xmax; + state->preprocess = preprocess_signed; + } else { + state->xmin = 0; + state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample); + state->preprocess = preprocess_unsigned; + } + + state->kmax = (1U << state->id_len) - 3; + + state->data_pp = malloc(strm->rsi + * strm->block_size + * sizeof(uint32_t)); + if (state->data_pp == NULL) { + cleanup(strm); + return AEC_MEM_ERROR; + } + + if (strm->flags & AEC_DATA_PREPROCESS) { + state->data_raw = malloc(strm->rsi + * strm->block_size + * sizeof(uint32_t)); + if (state->data_raw == NULL) { + cleanup(strm); + return AEC_MEM_ERROR; + } + } else { + state->data_raw = state->data_pp; + } + + state->block = state->data_pp; + + state->ref = 0; + strm->total_in = 0; + strm->total_out = 0; + state->flushed = 0; + + state->cds = state->cds_buf; + *state->cds = 0; + state->bits = 8; + state->mode = m_get_block; + + return AEC_OK; +} + +int aec_encode(struct aec_stream *strm, int flush) +{ + /** + Finite-state machine implementation of the adaptive entropy + encoder. + */ + struct internal_state *state = strm->state; + + state->flush = flush; + strm->total_in += strm->avail_in; + strm->total_out += strm->avail_out; + + while (state->mode(strm) == M_CONTINUE); + + if (state->direct_out) { + int n = (int)(state->cds - strm->next_out); + strm->next_out += n; + strm->avail_out -= n; + + *state->cds_buf = *state->cds; + state->cds = state->cds_buf; + state->direct_out = 0; + } + strm->total_in -= strm->avail_in; + strm->total_out -= strm->avail_out; + return AEC_OK; +} + +int aec_encode_end(struct aec_stream *strm) +{ + struct internal_state *state = strm->state; + + int status = AEC_OK; + if (state->flush == AEC_FLUSH && state->flushed == 0) + status = AEC_STREAM_ERROR; + cleanup(strm); + return status; +} + +int aec_buffer_encode(struct aec_stream *strm) +{ + int status = aec_encode_init(strm); + if (status != AEC_OK) + return status; + status = aec_encode(strm, AEC_FLUSH); + if (status != AEC_OK) { + cleanup(strm); + return status; + } + return aec_encode_end(strm); +} diff --git a/src/encode.h b/src/encode.h new file mode 100644 index 0000000..9622c44 --- /dev/null +++ b/src/encode.h @@ -0,0 +1,160 @@ +/** + * @file encode.h + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Adaptive Entropy Encoder + * Based on CCSDS documents 121.0-B-2 and 120.0-G-3 + * + */ + +#ifndef ENCODE_H +#define ENCODE_H 1 + +#include + +#if HAVE_STDINT_H +# include +#endif + +#define M_CONTINUE 1 +#define M_EXIT 0 +#define MIN(a, b) (((a) < (b))? (a): (b)) + +/* Maximum CDS length in bytes: 5 bits ID, 64 * 32 bits samples, 7 + * bits carry from previous CDS */ +#define CDSLEN ((5 + 64 * 32 + 7 + 7) / 8) + +/* Marker for Remainder Of Segment condition in zero block encoding */ +#define ROS -1 + +struct aec_stream; + +struct internal_state { + int (*mode)(struct aec_stream *); + uint32_t (*get_sample)(struct aec_stream *); + void (*get_rsi)(struct aec_stream *); + void (*preprocess)(struct aec_stream *); + + /* bit length of code option identification key */ + int id_len; + + /* minimum integer for preprocessing */ + uint32_t xmin; + + /* maximum integer for preprocessing */ + uint32_t xmax; + + uint32_t i; + + /* RSI blocks of preprocessed input */ + uint32_t *data_pp; + + /* RSI blocks of input */ + uint32_t *data_raw; + + /* remaining blocks in buffer */ + int blocks_avail; + + /* blocks encoded so far in RSI */ + int blocks_dispensed; + + /* current (preprocessed) input block */ + uint32_t *block; + + /* reference sample interval in byte */ + uint32_t rsi_len; + + /* current Coded Data Set output */ + uint8_t *cds; + + /* buffer for one CDS (only used if strm->next_out cannot hold + * full CDS) */ + uint8_t cds_buf[CDSLEN]; + + /* cds points to strm->next_out (1) or cds_buf (0) */ + int direct_out; + + /* Free bits (LSB) in output buffer or accumulator */ + int bits; + + /* length of reference sample in current block i.e. 0 or 1 + * depending on whether the block has a reference sample or not */ + int ref; + + /* reference sample stored here for performance reasons */ + uint32_t ref_sample; + + /* current zero block has a reference sample */ + int zero_ref; + + /* reference sample of zero block */ + uint32_t zero_ref_sample; + + /* storage size of samples in bytes */ + uint32_t bytes_per_sample; + + /* number of contiguous zero blocks */ + int zero_blocks; + + /* 1 if this is the first non-zero block after one or more zero + * blocks */ + int block_nonzero; + + /* splitting position */ + int k; + + /* maximum number for k depending on id_len */ + int kmax; + + /* flush option copied from argument */ + int flush; + + /* 1 if flushing was successful */ + int flushed; + + /* length of uncompressed CDS */ + uint32_t uncomp_len; +}; + +#endif /* ENCODE_H */ diff --git a/src/encode_accessors.c b/src/encode_accessors.c new file mode 100644 index 0000000..a3ef9b9 --- /dev/null +++ b/src/encode_accessors.c @@ -0,0 +1,250 @@ +/** + * @file encode_accessors.c + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Read various data types from input stream + * + */ + +#include + +#if HAVE_STDINT_H +# include +#endif + +#include +#include "libaec.h" +#include "encode.h" +#include "encode_accessors.h" + +uint32_t aec_get_8(struct aec_stream *strm) +{ + strm->avail_in--; + return *strm->next_in++; +} + +uint32_t aec_get_lsb_16(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[1] << 8) + | (uint32_t)strm->next_in[0]; + + strm->next_in += 2; + strm->avail_in -= 2; + return data; +} + +uint32_t aec_get_msb_16(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[0] << 8) + | (uint32_t)strm->next_in[1]; + + strm->next_in += 2; + strm->avail_in -= 2; + return data; +} + +uint32_t aec_get_lsb_24(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[2] << 16) + | ((uint32_t)strm->next_in[1] << 8) + | (uint32_t)strm->next_in[0]; + + strm->next_in += 3; + strm->avail_in -= 3; + return data; +} + +uint32_t aec_get_msb_24(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[0] << 16) + | ((uint32_t)strm->next_in[1] << 8) + | (uint32_t)strm->next_in[2]; + + strm->next_in += 3; + strm->avail_in -= 3; + return data; +} + +uint32_t aec_get_lsb_32(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[3] << 24) + | ((uint32_t)strm->next_in[2] << 16) + | ((uint32_t)strm->next_in[1] << 8) + | (uint32_t)strm->next_in[0]; + + strm->next_in += 4; + strm->avail_in -= 4; + return data; +} + +uint32_t aec_get_msb_32(struct aec_stream *strm) +{ + uint32_t data = ((uint32_t)strm->next_in[0] << 24) + | ((uint32_t)strm->next_in[1] << 16) + | ((uint32_t)strm->next_in[2] << 8) + | (uint32_t)strm->next_in[3]; + + strm->next_in += 4; + strm->avail_in -= 4; + return data; +} + +void aec_get_rsi_8(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + unsigned const char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = (uint32_t)in[i]; + + strm->next_in += rsi; + strm->avail_in -= rsi; +} + +void aec_get_rsi_lsb_16(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = (uint32_t)in[2 * i] | ((uint32_t)in[2 * i + 1] << 8); + + strm->next_in += 2 * rsi; + strm->avail_in -= 2 * rsi; +} + +void aec_get_rsi_msb_16(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = ((uint32_t)in[2 * i] << 8) | (uint32_t)in[2 * i + 1]; + + strm->next_in += 2 * rsi; + strm->avail_in -= 2 * rsi; +} + +void aec_get_rsi_lsb_24(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = (uint32_t)in[3 * i] + | ((uint32_t)in[3 * i + 1] << 8) + | ((uint32_t)in[3 * i + 2] << 16); + + strm->next_in += 3 * rsi; + strm->avail_in -= 3 * rsi; +} + +void aec_get_rsi_msb_24(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = ((uint32_t)in[3 * i] << 16) + | ((uint32_t)in[3 * i + 1] << 8) + | (uint32_t)in[3 * i + 2]; + + strm->next_in += 3 * rsi; + strm->avail_in -= 3 * rsi; +} + +#define AEC_GET_RSI_NATIVE_32(BO) \ + void aec_get_rsi_##BO##_32(struct aec_stream *strm) \ + { \ + int rsi = strm->rsi * strm->block_size; \ + memcpy(strm->state->data_raw, \ + strm->next_in, 4 * rsi); \ + strm->next_in += 4 * rsi; \ + strm->avail_in -= 4 * rsi; \ + } + +#ifdef WORDS_BIGENDIAN +void aec_get_rsi_lsb_32(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + for (int i = 0; i < rsi; i++) + out[i] = (uint32_t)in[4 * i] + | ((uint32_t)in[4 * i + 1] << 8) + | ((uint32_t)in[4 * i + 2] << 16) + | ((uint32_t)in[4 * i + 3] << 24); + + strm->next_in += 4 * rsi; + strm->avail_in -= 4 * rsi; +} + +AEC_GET_RSI_NATIVE_32(msb); + +#else /* !WORDS_BIGENDIAN */ +void aec_get_rsi_msb_32(struct aec_stream *strm) +{ + uint32_t *restrict out = strm->state->data_raw; + const unsigned char *restrict in = strm->next_in; + int rsi = strm->rsi * strm->block_size; + + strm->next_in += 4 * rsi; + strm->avail_in -= 4 * rsi; + + for (int i = 0; i < rsi; i++) + out[i] = ((uint32_t)in[4 * i] << 24) + | ((uint32_t)in[4 * i + 1] << 16) + | ((uint32_t)in[4 * i + 2] << 8) + | (uint32_t)in[4 * i + 3]; +} + +AEC_GET_RSI_NATIVE_32(lsb) + +#endif /* !WORDS_BIGENDIAN */ diff --git a/src/encode_accessors.h b/src/encode_accessors.h new file mode 100644 index 0000000..5c345ac --- /dev/null +++ b/src/encode_accessors.h @@ -0,0 +1,73 @@ +/** + * @file encode_accessors.h + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Read various data types from input stream + * + */ + +#ifndef ENCODE_ACCESSORS_H +#define ENCODE_ACCESSORS_H 1 + +#if HAVE_STDINT_H +# include +#endif + +uint32_t aec_get_8(struct aec_stream *strm); +uint32_t aec_get_lsb_16(struct aec_stream *strm); +uint32_t aec_get_msb_16(struct aec_stream *strm); +uint32_t aec_get_lsb_32(struct aec_stream *strm); +uint32_t aec_get_msb_24(struct aec_stream *strm); +uint32_t aec_get_lsb_24(struct aec_stream *strm); +uint32_t aec_get_msb_32(struct aec_stream *strm); + +void aec_get_rsi_8(struct aec_stream *strm); +void aec_get_rsi_lsb_16(struct aec_stream *strm); +void aec_get_rsi_msb_16(struct aec_stream *strm); +void aec_get_rsi_lsb_24(struct aec_stream *strm); +void aec_get_rsi_msb_24(struct aec_stream *strm); +void aec_get_rsi_lsb_32(struct aec_stream *strm); +void aec_get_rsi_msb_32(struct aec_stream *strm); + +#endif /* ENCODE_ACCESSORS_H */ diff --git a/src/libaec.h b/src/libaec.h new file mode 100644 index 0000000..4025532 --- /dev/null +++ b/src/libaec.h @@ -0,0 +1,176 @@ +/** + * @file libaec.h + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Adaptive Entropy Coding library + * + */ + +#ifndef LIBAEC_H +#define LIBAEC_H 1 + +#include + +#ifdef __cplusplus +extern "C"{ +#endif + +#if BUILDING_LIBAEC && HAVE_VISIBILITY +# define LIBAEC_DLL_EXPORTED __attribute__((__visibility__("default"))) +#elif BUILDING_LIBAEC && defined _MSC_VER +# define LIBAEC_DLL_EXPORTED __declspec(dllexport) +#elif defined _MSC_VER +# define LIBAEC_DLL_EXPORTED __declspec(dllimport) +#else +# define LIBAEC_DLL_EXPORTED +#endif + +struct internal_state; + +struct aec_stream { + const unsigned char *next_in; + + /* number of bytes available at next_in */ + size_t avail_in; + + /* total number of input bytes read so far */ + size_t total_in; + + unsigned char *next_out; + + /* remaining free space at next_out */ + size_t avail_out; + + /* total number of bytes output so far */ + size_t total_out; + + /* resolution in bits per sample (n = 1, ..., 32) */ + unsigned int bits_per_sample; + + /* block size in samples */ + unsigned int block_size; + + /* Reference sample interval, the number of Coded Data Sets + * between consecutive reference samples (up to 4096). */ + unsigned int rsi; + + unsigned int flags; + + struct internal_state *state; +}; + +/*********************************/ +/* Sample data description flags */ +/*********************************/ + +/* Samples are signed. Telling libaec this results in a slightly + * better compression ratio. Default is unsigned. */ +#define AEC_DATA_SIGNED 1 + +/* 24 bit samples are coded in 3 bytes */ +#define AEC_DATA_3BYTE 2 + +/* Samples are stored with their most significant bit first. This has + * nothing to do with the endianness of the host. Default is LSB. */ +#define AEC_DATA_MSB 4 + +/* Set if preprocessor should be used */ +#define AEC_DATA_PREPROCESS 8 + +/* Use restricted set of code options */ +#define AEC_RESTRICTED 16 + +/* Pad RSI to byte boundary. Only for decoding CCSDS sample data. */ +#define AEC_PAD_RSI 32 + +/* Do not enforce standard regarding legal block sizes. */ +#define AEC_NOT_ENFORCE 64 + +/*************************************/ +/* Return codes of library functions */ +/*************************************/ +#define AEC_OK 0 +#define AEC_CONF_ERROR (-1) +#define AEC_STREAM_ERROR (-2) +#define AEC_DATA_ERROR (-3) +#define AEC_MEM_ERROR (-4) + +/************************/ +/* Options for flushing */ +/************************/ + +/* Do not enforce output flushing. More input may be provided with + * later calls. So far only relevant for encoding. */ +#define AEC_NO_FLUSH 0 + +/* Flush output and end encoding. The last call to aec_encode() must + * set AEC_FLUSH to drain all output. + * + * It is not possible to continue encoding of the same stream after it + * has been flushed because the last byte may be padded with fill + * bits. */ +#define AEC_FLUSH 1 + +/*********************************************/ +/* Streaming encoding and decoding functions */ +/*********************************************/ +LIBAEC_DLL_EXPORTED int aec_encode_init(struct aec_stream *strm); +LIBAEC_DLL_EXPORTED int aec_encode(struct aec_stream *strm, int flush); +LIBAEC_DLL_EXPORTED int aec_encode_end(struct aec_stream *strm); + +LIBAEC_DLL_EXPORTED int aec_decode_init(struct aec_stream *strm); +LIBAEC_DLL_EXPORTED int aec_decode(struct aec_stream *strm, int flush); +LIBAEC_DLL_EXPORTED int aec_decode_end(struct aec_stream *strm); + +/***************************************************************/ +/* Utility functions for encoding or decoding a memory buffer. */ +/***************************************************************/ +LIBAEC_DLL_EXPORTED int aec_buffer_encode(struct aec_stream *strm); +LIBAEC_DLL_EXPORTED int aec_buffer_decode(struct aec_stream *strm); + +#ifdef __cplusplus +} +#endif + +#endif /* LIBAEC_H */ diff --git a/src/sz_compat.c b/src/sz_compat.c new file mode 100644 index 0000000..f852ed2 --- /dev/null +++ b/src/sz_compat.c @@ -0,0 +1,261 @@ +#include +#include +#include +#if HAVE_CONFIG_H +# include +#endif +#include "szlib.h" + +#define NOPTS 129 +#define MIN(a, b) (((a) < (b))? (a): (b)) + +static int convert_options(int sz_opts) +{ + int co[NOPTS]; + int opts = 0; + + memset(co, 0, sizeof(int) * NOPTS); + co[SZ_MSB_OPTION_MASK] = AEC_DATA_MSB; + co[SZ_NN_OPTION_MASK] = AEC_DATA_PREPROCESS; + + for (int i = 1; i < NOPTS; i <<= 1) + if (sz_opts & i) + opts |= co[i]; + + return opts; +} + +static int bits_to_bytes(int bit_length) +{ + if (bit_length > 16) + return 4; + else if (bit_length > 8) + return 2; + else + return 1; +} + +static void interleave_buffer(void *dest, const void *src, + size_t n, int wordsize) +{ + const unsigned char *src8 = (unsigned char *)src; + unsigned char *dest8 = (unsigned char *)dest; + + for (size_t i = 0; i < n / wordsize; i++) + for (size_t j = 0; j < wordsize; j++) + dest8[j * (n / wordsize) + i] = src8[i * wordsize + j]; +} + +static void deinterleave_buffer(void *dest, const void *src, + size_t n, int wordsize) +{ + const unsigned char *src8 = (unsigned char *)src; + unsigned char *dest8 = (unsigned char *)dest; + + for (size_t i = 0; i < n / wordsize; i++) + for (size_t j = 0; j < wordsize; j++) + dest8[i * wordsize + j] = src8[j * (n / wordsize) + i]; +} + +static void add_padding(void *dest, const void *src, size_t src_length, + size_t line_size, size_t padding_size, + int pixel_size, int pp) +{ + const char zero_pixel[] = {0, 0, 0, 0}; + + const char *pixel = zero_pixel; + size_t j = 0; + size_t i = 0; + while (i < src_length) { + size_t ps; + size_t ls = MIN(src_length - i, line_size); + memcpy((char *)dest + j, (char *)src + i, ls); + j += ls; + i += ls; + if (pp) + pixel = (char *)src + i - pixel_size; + ps = line_size + padding_size - ls; + for (size_t k = 0; k < ps; k += pixel_size) + memcpy((char *)dest + j + k, pixel, pixel_size); + j += ps; + } +} + +static void remove_padding(void *buf, size_t buf_length, + size_t line_size, size_t padding_size, + int pixel_size) +{ + size_t padded_line_size = line_size + padding_size; + + size_t i = line_size; + for (size_t j = padded_line_size; j < buf_length; j += padded_line_size) { + memmove((char *)buf + i, (char *)buf + j, line_size); + i += line_size; + } +} + +int SZ_BufftoBuffCompress(void *dest, size_t *destLen, + const void *source, size_t sourceLen, + SZ_com_t *param) +{ + struct aec_stream strm; + void *buf = 0; + void *padbuf = 0; + int status; + int interleave; + int pixel_size; + int aec_status; + size_t scanlines; + size_t padbuf_size; + size_t padding_size; + + strm.block_size = param->pixels_per_block; + strm.rsi = (param->pixels_per_scanline + param->pixels_per_block - 1) + / param->pixels_per_block; + strm.flags = AEC_NOT_ENFORCE | convert_options(param->options_mask); + strm.avail_out = *destLen; + strm.next_out = dest; + + interleave = param->bits_per_pixel == 32 || param->bits_per_pixel == 64; + if (interleave) { + strm.bits_per_sample = 8; + buf = malloc(sourceLen); + if (buf == NULL) { + status = SZ_MEM_ERROR; + goto CLEANUP; + } + interleave_buffer(buf, source, sourceLen, param->bits_per_pixel / 8); + } else { + strm.bits_per_sample = param->bits_per_pixel; + buf = (void *)source; + } + + pixel_size = bits_to_bytes(strm.bits_per_sample); + + scanlines = (sourceLen / pixel_size + param->pixels_per_scanline - 1) + / param->pixels_per_scanline; + padbuf_size = strm.rsi * strm.block_size * pixel_size * scanlines; + padbuf = malloc(padbuf_size); + if (padbuf == NULL) { + status = SZ_MEM_ERROR; + goto CLEANUP; + } + + padding_size = + (strm.rsi * strm.block_size - param->pixels_per_scanline) + * pixel_size; + + add_padding(padbuf, buf, sourceLen, + param->pixels_per_scanline * pixel_size, + padding_size, pixel_size, + strm.flags & AEC_DATA_PREPROCESS); + strm.next_in = padbuf; + strm.avail_in = padbuf_size; + + aec_status = aec_buffer_encode(&strm); + if (aec_status == AEC_STREAM_ERROR) + status = SZ_OUTBUFF_FULL; + else + status = aec_status; + *destLen = strm.total_out; + +CLEANUP: + if (padbuf) + free(padbuf); + if (interleave && buf) + free(buf); + return status; +} + +int SZ_BufftoBuffDecompress(void *dest, size_t *destLen, + const void *source, size_t sourceLen, + SZ_com_t *param) +{ + struct aec_stream strm; + void *buf = 0; + int status; + int pad_scanline; + int deinterleave; + int extra_buffer; + int pixel_size; + size_t total_out; + size_t scanlines; + + strm.block_size = param->pixels_per_block; + strm.rsi = (param->pixels_per_scanline + param->pixels_per_block - 1) + / param->pixels_per_block; + strm.flags = convert_options(param->options_mask); + strm.avail_in = sourceLen; + strm.next_in = source; + + pad_scanline = param->pixels_per_scanline % param->pixels_per_block; + deinterleave = (param->bits_per_pixel == 32 + || param->bits_per_pixel == 64); + extra_buffer = pad_scanline || deinterleave; + + if (deinterleave) + strm.bits_per_sample = 8; + else + strm.bits_per_sample = param->bits_per_pixel; + pixel_size = bits_to_bytes(strm.bits_per_sample); + + + if (extra_buffer) { + size_t buf_size; + if (pad_scanline) { + scanlines = (*destLen / pixel_size + param->pixels_per_scanline - 1) + / param->pixels_per_scanline; + buf_size = strm.rsi * strm.block_size * pixel_size * scanlines; + } else { + buf_size = *destLen; + } + buf = malloc(buf_size); + if (buf == NULL) { + status = SZ_MEM_ERROR; + goto CLEANUP; + } + strm.next_out = buf; + strm.avail_out = buf_size; + } else { + strm.next_out = dest; + strm.avail_out = *destLen; + } + + status = aec_buffer_decode(&strm); + if (status != AEC_OK) + goto CLEANUP; + + if (pad_scanline) { + size_t padding_size = + (strm.rsi * strm.block_size - param->pixels_per_scanline) + * pixel_size; + remove_padding(buf, strm.total_out, + param->pixels_per_scanline * pixel_size, + padding_size, pixel_size); + total_out = scanlines * param->pixels_per_scanline * pixel_size; + } else { + total_out = strm.total_out; + } + + if (total_out < *destLen) + *destLen = total_out; + + if (deinterleave) + deinterleave_buffer(dest, buf, *destLen, param->bits_per_pixel / 8); + else if (pad_scanline) + memcpy(dest, buf, *destLen); + +CLEANUP: + if (extra_buffer && buf) + free(buf); + + return status; +} + +int SZ_encoder_enabled(void) +{ + return 1; +} + +/* netcdf searches for SZ_Compress in configure */ +char SZ_Compress() { return SZ_OK; } diff --git a/src/szlib.h b/src/szlib.h new file mode 100644 index 0000000..c22362a --- /dev/null +++ b/src/szlib.h @@ -0,0 +1,45 @@ +#ifndef SZLIB_H +#define SZLIB_H 1 + +#include "libaec.h" + +#define SZ_ALLOW_K13_OPTION_MASK 1 +#define SZ_CHIP_OPTION_MASK 2 +#define SZ_EC_OPTION_MASK 4 +#define SZ_LSB_OPTION_MASK 8 +#define SZ_MSB_OPTION_MASK 16 +#define SZ_NN_OPTION_MASK 32 +#define SZ_RAW_OPTION_MASK 128 + +#define SZ_OK AEC_OK +#define SZ_OUTBUFF_FULL 2 + +#define SZ_NO_ENCODER_ERROR -1 +#define SZ_PARAM_ERROR AEC_CONF_ERROR +#define SZ_MEM_ERROR AEC_MEM_ERROR + +#define SZ_MAX_PIXELS_PER_BLOCK 32 +#define SZ_MAX_BLOCKS_PER_SCANLINE 128 +#define SZ_MAX_PIXELS_PER_SCANLINE \ + (SZ_MAX_BLOCKS_PER_SCANLINE) * (SZ_MAX_PIXELS_PER_BLOCK) + +typedef struct SZ_com_t_s +{ + int options_mask; + int bits_per_pixel; + int pixels_per_block; + int pixels_per_scanline; +} SZ_com_t; + +LIBAEC_DLL_EXPORTED int SZ_BufftoBuffCompress( + void *dest, size_t *destLen, + const void *source, size_t sourceLen, + SZ_com_t *param); +LIBAEC_DLL_EXPORTED int SZ_BufftoBuffDecompress( + void *dest, size_t *destLen, + const void *source, size_t sourceLen, + SZ_com_t *param); + +LIBAEC_DLL_EXPORTED int SZ_encoder_enabled(void); + +#endif /* SZLIB_H */ diff --git a/src/utime.c b/src/utime.c new file mode 100644 index 0000000..a7f1e99 --- /dev/null +++ b/src/utime.c @@ -0,0 +1,123 @@ +/** + * @file utime.c + * + * @author Thomas Jahns, Deutsches Klimarechenzentrum + * + * @section LICENSE + * Copyright 2012 - 2017 + * + * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens + * Deutsches Klimarechenzentrum GmbH + * Bundesstr. 45a + * 20146 Hamburg Germany + * + * Luis Kornblueh + * Max-Planck-Institut fuer Meteorologie + * Bundesstr. 53 + * 20146 Hamburg + * Germany + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @section DESCRIPTION + * + * Simple timing command, since calling time(1) gives non-portable results. + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +static int +run_cmd(int argc, char *argv[]); + +int main(int argc, char **argv) +{ + struct timeval utime = { .tv_sec = 0, .tv_usec = 0 }; + int rstatus; + int status = 0; + if (argc > 1 && ((status = run_cmd(argc - 1, argv + 1)) >= 0)) + { + struct rusage usage; + if ((rstatus = getrusage(RUSAGE_CHILDREN, &usage) == -1)) + { + perror("resource usage statistics unavailable"); + } + else if (rstatus != 0) + { + fputs("an unknown error occurred\n", stderr); + return EXIT_FAILURE; + } + else + { + utime = usage.ru_utime; + } + } + else if (status) + { + fputs("could not fork child\n", stderr); + return EXIT_FAILURE; + } + fprintf(stderr, "%f\n", (utime.tv_sec * 1000000 + utime.tv_usec)/1000000.0); + return status; +} + +static int +run_cmd(int argc, char *argv[]) +{ + int status; + pid_t child_pid; + if (argc < 1) + return -1; + if ((child_pid = fork()) < 0) + status = -1; + else if (child_pid == 0) + { + /* child */ + execvp(argv[0], argv); + _exit(127); /* execvp should not have returned */ + } + else + { + while (waitpid(child_pid, &status, 0) < 0) + if (errno != EINTR) + { + status = -1; + break; + } + } + return status; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..3553ff5 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,22 @@ +add_library(check_aec STATIC check_aec.c) + +add_executable(check_code_options check_code_options.c) +target_link_libraries(check_code_options check_aec aec) +add_test(NAME check_code_options COMMAND check_code_options) +add_executable(check_buffer_sizes check_buffer_sizes.c) +target_link_libraries(check_buffer_sizes check_aec aec) +add_test(NAME check_buffer_sizes COMMAND check_buffer_sizes) +add_executable(check_long_fs check_long_fs.c) +target_link_libraries(check_long_fs check_aec aec) +add_test(NAME check_long_fs COMMAND check_long_fs) +add_executable(check_szcomp check_szcomp.c) +target_link_libraries(check_szcomp check_aec sz) +add_test(NAME check_szcomp + COMMAND check_szcomp ${PROJECT_SOURCE_DIR}/data/121B2TestData/ExtendedParameters/sar32bit.dat) + +if(UNIX) + add_test( + NAME sampledata.sh + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/sampledata.sh ${CMAKE_CURRENT_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endif(UNIX) diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 0000000..eb62414 --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,28 @@ +AUTOMAKE_OPTIONS = color-tests +AM_CPPFLAGS = -I$(top_srcdir)/src +TESTS = check_code_options check_buffer_sizes check_long_fs \ +szcomp.sh sampledata.sh +TEST_EXTENSIONS = .sh +CLEANFILES = test.dat test.rz +check_LTLIBRARIES = libcheck_aec.la +libcheck_aec_la_SOURCES = check_aec.c check_aec.h +check_PROGRAMS = check_code_options check_buffer_sizes check_long_fs \ +check_szcomp + +check_code_options_SOURCES = check_code_options.c check_aec.h \ +$(top_srcdir)/src/libaec.h + +check_buffer_sizes_SOURCES = check_buffer_sizes.c check_aec.h \ +$(top_srcdir)/src/libaec.h + +check_long_fs_SOURCES = check_long_fs.c check_aec.h \ +$(top_srcdir)/src/libaec.h + +check_szcomp_SOURCES = check_szcomp.c $(top_srcdir)/src/szlib.h + +LDADD = libcheck_aec.la $(top_builddir)/src/libaec.la +check_szcomp_LDADD = $(top_builddir)/src/libsz.la + +EXTRA_DIST = sampledata.sh szcomp.sh CMakeLists.txt + +szcomp.log: sampledata.log diff --git a/tests/check_aec.c b/tests/check_aec.c new file mode 100644 index 0000000..354f357 --- /dev/null +++ b/tests/check_aec.c @@ -0,0 +1,310 @@ +#include +#include +#include +#include "check_aec.h" + +static void out_lsb(unsigned char *dest, unsigned long long int val, int size) +{ + for (int i = 0; i < size; i++) + dest[i] = (unsigned char)(val >> (8 * i)); +} + +static void out_msb(unsigned char *dest, unsigned long long int val, int size) +{ + for (int i = 0; i < size; i++) + dest[i] = (unsigned char)(val >> (8 * (size - 1 - i))); +} + +int update_state(struct test_state *state) +{ + struct aec_stream *strm = state->strm; + + if (strm->bits_per_sample > 16) { + state->id_len = 5; + + if (strm->bits_per_sample <= 24 && strm->flags & AEC_DATA_3BYTE) { + state->bytes_per_sample = 3; + } else { + state->bytes_per_sample = 4; + } + } + else if (strm->bits_per_sample > 8) { + state->id_len = 4; + state->bytes_per_sample = 2; + } else { + state->id_len = 3; + state->bytes_per_sample = 1; + } + + if (strm->flags & AEC_DATA_MSB) + state->out = out_msb; + else + state->out = out_lsb; + + if (strm->flags & AEC_DATA_SIGNED) { + state->xmin = -(1LL << (strm->bits_per_sample - 1)); + state->xmax = (1ULL << (strm->bits_per_sample - 1)) - 1; + } else { + state->xmin = 0; + state->xmax = (1ULL << strm->bits_per_sample) - 1; + } + + return 0; +} + +int encode_decode_small(struct test_state *state) +{ + size_t compressed_size; + size_t n_in, avail_in, avail_out, total_out; + struct aec_stream *strm = state->strm; + + int status = aec_encode_init(strm); + if (status != AEC_OK) { + printf("Init failed.\n"); + return 99; + } + + n_in = 0; + avail_in = 1; + avail_out = 1; + total_out = 0; + strm->next_in = state->ubuf; + strm->avail_in = state->bytes_per_sample; + strm->avail_out = 1; + strm->next_out = state->cbuf; + + while ((avail_in || avail_out) && total_out < state->cbuf_len) { + if (strm->avail_in == 0 && avail_in) { + n_in += state->bytes_per_sample; + if (n_in < state->buf_len) { + strm->avail_in = state->bytes_per_sample; + strm->next_in = state->ubuf + n_in; + } else { + avail_in = 0; + } + } + + status = aec_encode(strm, AEC_NO_FLUSH); + if (status != AEC_OK) { + printf("Decode failed.\n"); + return 99; + } + + if (strm->total_out - total_out > 0 + && total_out < state->cbuf_len) { + total_out = strm->total_out; + strm->avail_out = 1; + strm->next_out = state->cbuf + total_out; + avail_out = 1; + } else { + avail_out = 0; + } + } + + status = aec_encode(strm, AEC_FLUSH); + if (status != AEC_OK) { + printf("Encode failed.\n"); + return 99; + } + + aec_encode_end(strm); + + compressed_size = strm->total_out; + + strm->avail_in = 1; + strm->next_in = state->cbuf; + + strm->avail_out = state->bytes_per_sample; + strm->next_out = state->obuf; + + status = aec_decode_init(strm); + if (status != AEC_OK) { + printf("Init failed.\n"); + return 99; + } + + n_in = 0; + avail_in = 1; + avail_out = 1; + total_out = 0; + strm->next_in = state->cbuf; + strm->avail_in = 1; + strm->avail_out = state->bytes_per_sample; + strm->next_out = state->obuf; + + while ((avail_in || avail_out) && total_out < state->buf_len) { + if (strm->avail_in == 0 && avail_in) { + n_in++; + if (n_in < compressed_size) { + strm->avail_in = 1; + strm->next_in = state->cbuf + n_in; + } else { + avail_in = 0; + } + } + + status = aec_decode(strm, AEC_NO_FLUSH); + if (status != AEC_OK) { + printf("Decode failed.\n"); + return 99; + } + + if (strm->total_out - total_out > 0 + && total_out < state->buf_len) { + total_out = strm->total_out; + strm->avail_out = state->bytes_per_sample; + strm->next_out = state->obuf + total_out; + avail_out = 1; + } else { + avail_out = 0; + } + } + + status = aec_decode(strm, AEC_FLUSH); + if (status != AEC_OK) { + printf("Decode failed.\n"); + return 99; + } + + if (memcmp(state->ubuf, state->obuf, state->ibuf_len)) { + printf("\n%s: Uncompressed output differs from input.\n", CHECK_FAIL); + + printf("\nuncompressed buf"); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n"); + printf("%02x ", state->ubuf[i]); + } + printf("\n\ncompressed buf len %zu", compressed_size); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n"); + printf("%02x ", state->cbuf[i]); + } + printf("\n\ndecompressed buf"); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n%04i ", i); + printf("%02x ", state->obuf[i]); + } + printf("\n"); + return 99; + } + aec_decode_end(strm); + return 0; +} + +int encode_decode_large(struct test_state *state) +{ + int status; + int bflags = 0; + size_t to; + char fbase[1024]; + struct aec_stream *strm = state->strm; + + strm->avail_in = state->ibuf_len; + strm->avail_out = state->cbuf_len; + strm->next_in = state->ubuf; + strm->next_out = state->cbuf; + + status = aec_encode_init(strm); + if (status != AEC_OK) { + printf("Init failed.\n"); + return 99; + } + if (state->dump) { + char fname[1024]; + FILE *fp; + snprintf(fbase, sizeof(fbase), "BPS%02iID%iBS%02iRSI%04iFLG%04i", + strm->bits_per_sample, + state->id, + strm->block_size, + strm->rsi, + strm->flags); + snprintf(fname, sizeof(fname), "%s.dat", fbase); + if ((fp = fopen(fname, "wb")) == NULL) { + fprintf(stderr, "ERROR: cannot open dump file %s\n", fname); + return 99; + } + fputc(strm->bits_per_sample, fp); + bflags = strm->block_size >> 8; + if (strm->flags | AEC_DATA_MSB) + bflags |= 0x80; + if (strm->flags | AEC_DATA_SIGNED) + bflags |= 0x40; + if (strm->flags | AEC_DATA_3BYTE) + bflags |= 0x10; + bflags |= 0x20; /* encode */ + fputc(bflags, fp); + fwrite(strm->next_in, strm->avail_in, 1, fp); + fclose(fp); + } + + status = aec_encode(strm, AEC_FLUSH); + if (status != AEC_OK) { + printf("Encode failed.\n"); + return 99; + } + + aec_encode_end(strm); + + if (state->dump) { + char fname[1024]; + FILE *fp; + snprintf(fname, sizeof(fname), "%s.rz", fbase); + if ((fp = fopen(fname, "wb")) == NULL) { + fprintf(stderr, "ERROR: cannot open dump file %s\n", fname); + return 99; + } + fputc(strm->bits_per_sample, fp); + bflags &= ~0x20; + fputc(bflags, fp); + fwrite(state->cbuf, strm->total_out, 1, fp); + fclose(fp); + } + + strm->avail_in = strm->total_out; + strm->avail_out = state->buf_len; + strm->next_in = state->cbuf; + strm->next_out = state->obuf; + to = strm->total_out; + + status = aec_decode_init(strm); + if (status != AEC_OK) { + printf("Init failed.\n"); + return 99; + } + + status = aec_decode(strm, AEC_FLUSH); + if (status != AEC_OK) { + printf("Decode failed.\n"); + return 99; + } + + if (memcmp(state->ubuf, state->obuf, state->ibuf_len)) { + printf("\n%s: Uncompressed output differs from input.\n", CHECK_FAIL); + + printf("\nuncompressed buf"); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n"); + printf("%02x ", state->ubuf[i]); + } + printf("\n\ncompressed buf len %zu", to); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n"); + printf("%02x ", state->cbuf[i]); + } + printf("\n\ndecompressed buf"); + for (int i = 0; i < 80; i++) { + if (i % 8 == 0) + printf("\n"); + printf("%02x ", state->obuf[i]); + } + printf("\n"); + return 99; + } + aec_decode_end(strm); + return 0; +} diff --git a/tests/check_aec.h b/tests/check_aec.h new file mode 100644 index 0000000..2f6abd3 --- /dev/null +++ b/tests/check_aec.h @@ -0,0 +1,49 @@ +#ifndef CHECK_AEC_H +#define CHECK_AEC_H 1 + +#include +#include "libaec.h" + +struct test_state { + int (* codec)(struct test_state *state); + int id; + int id_len; + int bytes_per_sample; + unsigned char *ubuf; + unsigned char *cbuf; + unsigned char *obuf; + size_t ibuf_len; /* input buffer legth may be shorter than buf_len */ + size_t buf_len; + size_t cbuf_len; + long long int xmax; + long long int xmin; + void (*out)(unsigned char *dest, unsigned long long int val, int size); + int dump; /* dump buffer to file for fuzzing corpus */ + struct aec_stream *strm; +}; + +int update_state(struct test_state *state); +int encode_decode_small(struct test_state *state); +int encode_decode_large(struct test_state *state); + +#ifndef HAVE_SNPRINTF +#ifdef HAVE__SNPRINTF_S +#define snprintf(d, n, ...) _snprintf_s((d), (n), _TRUNCATE, __VA_ARGS__) +#else +#ifdef HAVE__SNPRINTF +#define snprintf _snprintf +#else +#error "no snprintf compatible function found" +#endif /* HAVE__SNPRINTF */ +#endif /* HAVE__SNPRINTF_S */ +#endif /* HAVE_SNPRINTF */ + +#ifdef _WIN32 +#define CHECK_PASS "PASS" +#define CHECK_FAIL "FAIL" +#else +#define CHECK_PASS "PASS" +#define CHECK_FAIL "FAIL" +#endif + +#endif /* CHECK_AEC_H */ diff --git a/tests/check_buffer_sizes.c b/tests/check_buffer_sizes.c new file mode 100644 index 0000000..53996a0 --- /dev/null +++ b/tests/check_buffer_sizes.c @@ -0,0 +1,108 @@ +#include +#include +#include +#include "check_aec.h" + +#define BUF_SIZE 1024 * 3 + +int check_block_sizes(struct test_state *state) +{ + for (int bs = 8; bs <= 64; bs *= 2) { + int status; + state->strm->block_size = bs; + state->strm->rsi = (int)(state->buf_len + / (bs * state->bytes_per_sample)); + + status = encode_decode_large(state); + if (status) + return status; + } + return 0; +} + +int check_block_sizes_short(struct test_state *state) +{ + size_t tmp = state->ibuf_len; + for (int bs = 8; bs <= 64; bs *= 2) { + int status; + state->strm->block_size = bs; + state->strm->rsi = (int)(state->buf_len + / (bs * state->bytes_per_sample)); + state->ibuf_len = state->buf_len - 2 * bs + 4; + status = encode_decode_large(state); + if (status) + return status; + if (state->strm->total_out != state->buf_len) { + printf("FAIL: Unexpected buffer length. Got %i expected %i\n", + (int)state->strm->total_out, + (int)state->buf_len); + return 99; + } + } + state->ibuf_len = tmp; + return 0; +} + +int check_rsi(struct test_state *state) +{ + int status; + int size = state->bytes_per_sample; + + for (unsigned char *tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 2 * state->bytes_per_sample) { + state->out(tmp, state->xmax, size); + state->out(tmp + size, state->xmin, size); + } + + printf("Checking full rsi ... "); + status = check_block_sizes(state); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + + printf("Checking short rsi ... "); + status = check_block_sizes_short(state); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int main (void) +{ + int status; + struct aec_stream strm; + struct test_state state; + + state.dump = 0; + state.buf_len = state.ibuf_len = BUF_SIZE; + state.cbuf_len = 2 * BUF_SIZE; + + state.ubuf = (unsigned char *)malloc(state.buf_len); + state.cbuf = (unsigned char *)malloc(state.cbuf_len); + state.obuf = (unsigned char *)malloc(state.buf_len); + + if (!state.ubuf || !state.cbuf || !state.obuf) { + printf("Not enough memory.\n"); + return 99; + } + + strm.flags = AEC_DATA_PREPROCESS; + state.strm = &strm; + strm.bits_per_sample = 32; + update_state(&state); + + status = check_rsi(&state); + if (status) + goto DESTRUCT; + +DESTRUCT: + free(state.ubuf); + free(state.cbuf); + free(state.obuf); + + return status; +} diff --git a/tests/check_code_options.c b/tests/check_code_options.c new file mode 100644 index 0000000..dea916f --- /dev/null +++ b/tests/check_code_options.c @@ -0,0 +1,332 @@ +#include +#include +#include +#include "check_aec.h" + +#define BUF_SIZE 1024 * 3 + +int check_block_sizes(struct test_state *state, int id, int id_len) +{ + state->id = id; + for (int bs = 8; bs <= 64; bs *= 2) { + int max_rsi; + + state->strm->block_size = bs; + max_rsi = (int)(state->buf_len / (bs * state->bytes_per_sample)); + if (max_rsi > 4096) + max_rsi = 4096; + + for (int rsi = 1; rsi <= max_rsi; rsi++) { + int status; + state->strm->rsi = rsi; + status = state->codec(state); + if (status) + return status; + + if ((state->cbuf[0] >> (8 - id_len)) != id) { + printf( + "%s: block of size %i created with ID:%x, expected %x.\n", + CHECK_FAIL, bs, state->cbuf[0] >> (8 - id_len), id + ); + return 99; + } + } + } + return 0; +} + +int check_zero(struct test_state *state) +{ + int status; + + if (state->strm->flags & AEC_DATA_PREPROCESS) + memset(state->ubuf, 0x55, state->buf_len); + else + memset(state->ubuf, 0, state->buf_len); + + printf("Checking zero blocks ... "); + status = check_block_sizes(state, 0, state->id_len + 1); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int check_splitting(struct test_state *state, int k) +{ + int status; + unsigned char *tmp; + + int size = state->bytes_per_sample; + + if (state->strm->flags & AEC_DATA_PREPROCESS) { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 4 * size) { + state->out(tmp, state->xmin + (1ULL << (k - 1)) - 1, size); + state->out(tmp + size, state->xmin, size); + state->out(tmp + 2 * size, state->xmin + + (1ULL << (k + 1)) - 1, size); + state->out(tmp + 3 * size, state->xmin, size); + } + } else { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 4 * size) { + state->out(tmp, 0, size); + state->out(tmp + size, (1ULL << k) - 1, size); + state->out(tmp + 2 * size, 0, size); + state->out(tmp + 3 * size, (1ULL << (k + 2)) - 1, size); + } + } + + printf("Checking splitting with k=%i ... ", k); + status = check_block_sizes(state, k + 1, state->id_len); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int check_uncompressed(struct test_state *state) +{ + int status; + unsigned char *tmp; + + int size = state->bytes_per_sample; + + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 2 * size) { + state->out(tmp, state->xmax, size); + state->out(tmp + size, state->xmin, size); + } + + printf("Checking uncompressed ... "); + status = check_block_sizes(state, + (1ULL << state->id_len) - 1, + state->id_len); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int check_fs(struct test_state *state) +{ + int status; + unsigned char *tmp; + + int size = state->bytes_per_sample; + + if (state->strm->flags & AEC_DATA_PREPROCESS) { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 4 * size) { + state->out(tmp, state->xmin + 2, size); + state->out(tmp + size, state->xmin, size); + state->out(tmp + 2 * size, state->xmin, size); + state->out(tmp + 3 * size, state->xmin, size); + } + } else { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 4 * size) { + state->out(tmp, 0, size); + state->out(tmp + size, 0, size); + state->out(tmp + 2 * size, 0, size); + state->out(tmp + 3 * size, 4, size); + } + } + + printf("Checking FS ... "); + status = check_block_sizes(state, 1, state->id_len); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int check_se(struct test_state *state) +{ + int status; + unsigned char *tmp; + + int size = state->bytes_per_sample; + + if (state->strm->flags & AEC_DATA_PREPROCESS) { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 8 * size) { + state->out(tmp, state->xmax - 1, size); + state->out(tmp + size, state->xmax - 1, size); + state->out(tmp + 2 * size, state->xmax - 1, size); + state->out(tmp + 3 * size, state->xmax - 1, size); + state->out(tmp + 4 * size, state->xmax, size); + state->out(tmp + 5 * size, state->xmax, size); + state->out(tmp + 6 * size, state->xmax, size); + state->out(tmp + 7 * size, state->xmax, size); + } + } else { + for (tmp = state->ubuf; + tmp < state->ubuf + state->buf_len; + tmp += 8 * size) { + state->out(tmp, 0, size); + state->out(tmp + size, 0, size); + state->out(tmp + 2 * size, 0, size); + state->out(tmp + 3 * size, 0, size); + state->out(tmp + 4 * size, 1, size); + state->out(tmp + 5 * size, 0, size); + state->out(tmp + 6 * size, 0, size); + state->out(tmp + 7 * size, 2, size); + } + } + + printf("Checking Second Extension ... "); + status = check_block_sizes(state, 1, state->id_len + 1); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int check_bps(struct test_state *state) +{ + int status; + + for (int bps = 8; bps <= 32; bps += 8) { + state->strm->bits_per_sample = bps; + if (bps == 24) + state->strm->flags |= AEC_DATA_3BYTE; + else + state->strm->flags &= ~AEC_DATA_3BYTE; + + update_state(state); + + status = check_zero(state); + if (status) + return status; + + status = check_se(state); + if (status) + return status; + + status = check_uncompressed(state); + if (status) + return status; + + status = check_fs(state); + if (status) + return status; + + for (int k = 1; k < bps - 2; k++) { + status = check_splitting(state, k); + if (status) + return status; + } + printf("All checks with %i bit per sample passed.\n", bps); + } + return 0; +} + +int check_byte_orderings(struct test_state *state) +{ + int status; + + printf("-----------------------------------\n"); + printf("Checking no PP, LSB first, unsigned\n"); + printf("-----------------------------------\n"); + status = check_bps(state); + if (status) + return status; + + printf("-----------------------------------\n"); + printf("Checking PP, LSB first, unsigned\n"); + printf("-----------------------------------\n"); + state->strm->flags |= AEC_DATA_PREPROCESS; + status = check_bps(state); + if (status) + return status; + + printf("-----------------------------------\n"); + printf("Checking PP, LSB first, signed\n"); + printf("-----------------------------------\n"); + state->strm->flags |= AEC_DATA_SIGNED; + + status = check_bps(state); + if (status) + return status; + + state->strm->flags &= ~AEC_DATA_SIGNED; + state->strm->flags |= AEC_DATA_MSB; + + printf("-----------------------------------\n"); + printf("Checking PP, MSB first, unsigned\n"); + printf("-----------------------------------\n"); + status = check_bps(state); + if (status) + return status; + + printf("-----------------------------------\n"); + printf("Checking PP, MSB first, signed\n"); + printf("-----------------------------------\n"); + state->strm->flags |= AEC_DATA_SIGNED; + + status = check_bps(state); + if (status) + return status; + return 0; +} + +int main(int argc, char *argv[]) +{ + int status; + struct aec_stream strm; + struct test_state state; + + if (argc > 1 && strncmp(argv[1], "-d", 2) == 0) + state.dump = 1; + else + state.dump = 0; + + state.buf_len = state.ibuf_len = BUF_SIZE; + state.cbuf_len = 2 * BUF_SIZE; + + state.ubuf = (unsigned char *)malloc(state.buf_len); + state.cbuf = (unsigned char *)malloc(state.cbuf_len); + state.obuf = (unsigned char *)malloc(state.buf_len); + + if (!state.ubuf || !state.cbuf || !state.obuf) { + printf("Not enough memory.\n"); + return 99; + } + + strm.flags = 0; + state.strm = &strm; + + printf("***************************\n"); + printf("Checking with small buffers\n"); + printf("***************************\n"); + state.codec = encode_decode_small; + status = check_byte_orderings(&state); + if (status) + goto DESTRUCT; + + printf("***************************\n"); + printf("Checking with large buffers\n"); + printf("***************************\n"); + state.codec = encode_decode_large; + status = check_byte_orderings(&state); + +DESTRUCT: + free(state.ubuf); + free(state.cbuf); + free(state.obuf); + + return status; +} diff --git a/tests/check_long_fs.c b/tests/check_long_fs.c new file mode 100644 index 0000000..49bdaa3 --- /dev/null +++ b/tests/check_long_fs.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include "check_aec.h" + +#define BUF_SIZE (64 * 4) + +int check_long_fs(struct test_state *state) +{ + int status; + int size = state->bytes_per_sample; + int bs = state->strm->block_size; + + for (int i = 0; i < bs / 2; i++) { + state->out(state->ubuf + size * i, state->xmin, size); + state->out(state->ubuf + bs * size / 2 + size * i, 65000, size); + } + + printf("Checking long fs ... "); + + status = state->codec(state); + if (status) + return status; + + printf ("%s\n", CHECK_PASS); + return 0; +} + +int main (void) +{ + int status; + struct aec_stream strm; + struct test_state state; + + state.dump = 0; + state.buf_len = state.ibuf_len = BUF_SIZE; + state.cbuf_len = 2 * BUF_SIZE; + + state.ubuf = (unsigned char *)malloc(state.buf_len); + state.cbuf = (unsigned char *)malloc(state.cbuf_len); + state.obuf = (unsigned char *)malloc(state.buf_len); + + if (!state.ubuf || !state.cbuf || !state.obuf) { + printf("Not enough memory.\n"); + return 99; + } + + strm.flags = AEC_DATA_PREPROCESS; + state.strm = &strm; + strm.bits_per_sample = 16; + strm.block_size = 64; + strm.rsi = 1; + state.codec = encode_decode_large; + update_state(&state); + + status = check_long_fs(&state); + if (status) + goto DESTRUCT; + +DESTRUCT: + free(state.ubuf); + free(state.cbuf); + free(state.obuf); + + return status; +} diff --git a/tests/check_szcomp.c b/tests/check_szcomp.c new file mode 100644 index 0000000..8133754 --- /dev/null +++ b/tests/check_szcomp.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include "szlib.h" + +#define OPTIONS_MASK (SZ_RAW_OPTION_MASK \ + | SZ_MSB_OPTION_MASK \ + | SZ_NN_OPTION_MASK) +#define PIXELS_PER_BLOCK (8) +#define PIXELS_PER_SCANLINE (PIXELS_PER_BLOCK*128) + +int main(int argc, char *argv[]) +{ + int status; + SZ_com_t sz_param; + unsigned char *source, *dest, *dest1; + size_t destLen, dest1Len, sourceLen; + FILE *fp; + + if (argc < 2) { + fprintf(stderr, "Usage: %s file\n", argv[0]); + return 1; + } + + if ((fp = fopen(argv[1], "rb")) == NULL) { + fprintf(stderr, "Can't open %s\n", argv[1]); + return 1; + } + fseek(fp, 0L, SEEK_END); + sourceLen = ftell(fp); + fseek(fp, 0L, SEEK_SET); + destLen = sourceLen + sourceLen / 10; + + sz_param.options_mask = OPTIONS_MASK; + sz_param.bits_per_pixel = 64; + sz_param.pixels_per_block = PIXELS_PER_BLOCK; + sz_param.pixels_per_scanline = PIXELS_PER_SCANLINE; + + source = (unsigned char *)malloc(sourceLen); + dest = (unsigned char *)malloc(destLen); + dest1 = (unsigned char *)malloc(destLen); + + if (source == NULL || dest == NULL || dest1 == NULL) + return 1; + + sourceLen = fread(source, 1, sourceLen, fp); + + status = SZ_BufftoBuffCompress(dest, &destLen, + source, sourceLen, &sz_param); + if (status != SZ_OK) + return status; + + dest1Len = sourceLen; + status = SZ_BufftoBuffDecompress(dest1, &dest1Len, + dest, destLen, &sz_param); + if (status != SZ_OK) + return status; + + if (memcmp(source, dest1, sourceLen) != 0) + fprintf(stderr, "File %s Buffers differ\n", argv[2]); + + free(source); + free(dest); + free(dest1); + return 0; +} diff --git a/tests/sampledata.sh b/tests/sampledata.sh new file mode 100755 index 0000000..5729c8e --- /dev/null +++ b/tests/sampledata.sh @@ -0,0 +1,80 @@ +#!/bin/sh +set -e +AEC=../src/aec +if [ -n "$1" ]; then + srcdir=$1 +fi +CCSDS_DATA=${srcdir}/../data/121B2TestData +ALLO=${CCSDS_DATA}/AllOptions +EXTP=${CCSDS_DATA}/ExtendedParameters +LOWE=${CCSDS_DATA}/LowEntropyOptions + +filesize () { + wc -c $1 | awk '{print $1}' +} + +decode () { + $AEC -d $3 $1 test.dat + dd if=test.dat bs=1 count=$(filesize $2) | cmp $2 - +} + +code () { + $AEC $3 $2 test.rz + cmp $1 test.rz +} + +code_size () { + $AEC $3 $2 test.rz + if [ ! $(filesize test.rz) -eq $(filesize $1) ]; then + echo "$1 size mismatch" + exit 1 + fi +} + +codec () { + code "$@" + decode "$@" +} + +cosdec () { + code_size "$@" + decode "$@" +} + +echo All Options +for i in 01 02 03 04 +do + uf=$ALLO/test_p256n${i}.dat + codec $ALLO/test_p256n${i}-basic.rz $uf "-n$i -j16 -r16" + codec $ALLO/test_p256n${i}-restricted.rz $uf "-n$i -j16 -r16 -t" +done +for i in 05 06 07 08 09 10 11 12 13 14 15 16 +do + cosdec $ALLO/test_p256n${i}.rz $ALLO/test_p256n${i}.dat \ + "-n$i -j16 -r16" +done +for i in 17 18 19 20 21 22 23 24 +do + cosdec $ALLO/test_p512n${i}.rz $ALLO/test_p512n${i}.dat \ + "-n$i -j16 -r32" +done + +echo Low Entropy Options +for i in 1 2 3 +do + for j in 01 02 03 04 + do + uf=$LOWE/Lowset${i}_8bit.dat + codec $LOWE/Lowset${i}_8bit.n${j}-basic.rz $uf "-n$j -j16 -r64" + codec $LOWE/Lowset${i}_8bit.n${j}-restricted.rz $uf "-n$j -j16 -r64 -t" + done + for j in 05 06 07 08 + do + codec $LOWE/Lowset${i}_8bit.n${j}.rz $LOWE/Lowset${i}_8bit.dat \ + "-n$j -j16 -r64" + done +done + +echo Extended Parameters +decode $EXTP/sar32bit.j16.r256.rz $EXTP/sar32bit.dat "-n32 -j16 -r256 -p" +decode $EXTP/sar32bit.j64.r4096.rz $EXTP/sar32bit.dat "-n32 -j64 -r4096 -p" diff --git a/tests/szcomp.sh b/tests/szcomp.sh new file mode 100755 index 0000000..f140932 --- /dev/null +++ b/tests/szcomp.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e +testfile=${srcdir}/../data/121B2TestData/ExtendedParameters/sar32bit.dat +if [ ! -f $testfile ]; then + echo "ERROR: sample data not found." + exit -1 +fi +./check_szcomp $testfile