Add support for lz4hdf5 codec

Author: MarkRivers

ADApp/ADSrc/Codec.h

@@ -6,6 +6,7 @@ static std::string codecName[] = {
     "jpeg",
     "blosc",
     "lz4",
+    "lz4hdf5",
     "bslz4"
 };
 
@@ -14,6 +15,7 @@ typedef enum {
   NDCODEC_JPEG,
   NDCODEC_BLOSC,
   NDCODEC_LZ4,
+  NDCODEC_LZ4HDF5,
   NDCODEC_BSLZ4
 } NDCodecCompressor_t;
 

ADApp/Db/NDCodec.template

@@ -42,8 +42,10 @@ record(mbbo, "$(P)$(R)Compressor")
     field(TWVL, "2")
     field(THST, "LZ4")
     field(THVL, "3")
-    field(FRST, "BSLZ4")
+    field(FRST, "LZ4HDF5")
     field(FRVL, "4")
+    field(FVST, "BSLZ4")
+    field(FVVL, "5")
     info(autosaveFields, "VAL")
 }
 
@@ -59,8 +61,10 @@ record(mbbi, "$(P)$(R)Compressor_RBV")
     field(TWVL, "2")
     field(THST, "LZ4")
     field(THVL, "3")
-    field(FRST, "BSLZ4")
+    field(FRST, "LZ4HDF5")
     field(FRVL, "4")
+    field(FVST, "BSLZ4")
+    field(FVVL, "5")
     field(SCAN, "I/O Intr")
 }
 

ADApp/commonDriverMakefile

@@ -153,6 +153,7 @@ ifeq ($(WITH_BITSHUFFLE),YES)
       PROD_SYS_LIBS += bitshuffle
     endif
   endif
+  PROD_LIBS += lz4hdf5
 endif
 
 ifeq ($(WITH_BLOSC),YES)

ADApp/commonLibraryMakefile

@@ -118,6 +118,7 @@ ifeq ($(WITH_BITSHUFFLE),YES)
       LIB_SYS_LIBS += bitshuffle
     endif
   endif
+  LIB_LIBS += lz4hdf5
 endif
 
 ifeq ($(WITH_BLOSC),YES)

ADApp/pluginSrc/NDFileHDF5.cpp

@@ -40,6 +40,7 @@ enum HDF5Compression_t {HDF5CompressNone=0,
                         HDF5CompressBlosc,
                         HDF5CompressBshuf,
                         HDF5CompressLZ4,
+                        HDF5CompressLZ4HDF5,
                         HDF5CompressJPEG};
 /* Filter ID officially assigned to blosc */
 #define FILTER_BLOSC 32001
@@ -1940,6 +1941,7 @@ asynStatus NDFileHDF5::writeInt32(asynUser *pasynUser, epicsInt32 value)
         filterId = FILTER_BSHUF;
         break;
       case HDF5CompressLZ4:
+      case HDF5CompressLZ4HDF5:
         filterId = FILTER_LZ4;
         break;
       case HDF5CompressJPEG:
@@ -3325,6 +3327,8 @@ asynStatus NDFileHDF5::configureCompression(NDArray *pArray)
       setIntegerParam(NDFileHDF5_compressionType, HDF5CompressBshuf);
     } else if (pArray->codec.name == codecName[NDCODEC_LZ4]) {
       setIntegerParam(NDFileHDF5_compressionType, HDF5CompressLZ4);
+    } else if (pArray->codec.name == codecName[NDCODEC_LZ4HDF5]) {
+      setIntegerParam(NDFileHDF5_compressionType, HDF5CompressLZ4HDF5);
     } else if (pArray->codec.name == codecName[NDCODEC_JPEG]) {
       setIntegerParam(NDFileHDF5_compressionType, HDF5CompressJPEG);
     }
@@ -3411,7 +3415,8 @@ asynStatus NDFileHDF5::configureCompression(NDArray *pArray)
         this->codec.name = codecName[NDCODEC_BSLZ4];
       }
       break;
-    case HDF5CompressLZ4: {
+    case HDF5CompressLZ4:
+    case HDF5CompressLZ4HDF5: {
         unsigned int cds[2];
         cds[0] = 0; /* lz4 selects the block size automatically */
         cds[1] = 0; /* Number of threads (not implemented) */
@@ -3420,7 +3425,11 @@ asynStatus NDFileHDF5::configureCompression(NDArray *pArray)
           asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, "Failed to set h5 lz4 filter\n");
           break;
         }
-        this->codec.name = codecName[NDCODEC_LZ4];
+        if (compressionScheme == HDF5CompressLZ4) {
+          this->codec.name = codecName[NDCODEC_LZ4];
+        } else {
+          this->codec.name = codecName[NDCODEC_LZ4HDF5];
+        }
       }
       break;
     case HDF5CompressJPEG: {

ADApp/pluginSrc/NDPluginCodec.cpp

@@ -460,6 +460,7 @@ NDArray *decompressBlosc(NDArray *input, int numThreads, NDCodecStatus_t *status
 #ifdef HAVE_BITSHUFFLE
 #include <bitshuffle.h>
 #include <lz4.h>
+#include <lz4hdf5.h>
 
 NDArray *compressLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage)
 {
@@ -517,7 +518,7 @@ NDArray *decompressLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessa
         return NULL;
     }
 
-    int ret = LZ4_decompress_fast((const char*)input->pData, (char*)output->pData, (int)info.totalBytes);
+    int ret = LZ4_decompress_safe((const char*)input->pData, (char*)output->pData, (int)input->compressedSize, (int)info.totalBytes);
 
     if (ret <= 0){
         output->release();
@@ -531,6 +532,76 @@ NDArray *decompressLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessa
     return output;
 }
 
+NDArray *compressLZ4HDF5(NDArray *input, NDCodecStatus_t *status, char *errorMessage)
+{
+    if (!input->codec.empty()) {
+        sprintf(errorMessage, "Array is already compressed");
+        *status = NDCODEC_WARNING;
+        return NULL;
+    }
+
+    NDArrayInfo_t info;
+    input->getInfo(&info);
+    int outputSize = LZ4_compressBound((int)info.totalBytes);
+    NDArray *output = allocArray(input, -1, outputSize);
+
+    if (!output) {
+        sprintf(errorMessage, "Failed to allocate LZ4HDF5 output array");
+        *status = NDCODEC_ERROR;
+        return NULL;
+    }
+
+    int compSize = compress_lz4hdf5((const char*)input->pData, (char*)output->pData, info.totalBytes, outputSize);
+
+    if (compSize <= 0) {
+        output->release();
+        sprintf(errorMessage, "Internal LZ4HDF5 error");
+        *status = NDCODEC_ERROR;
+        return NULL;
+    }
+
+    output->codec.name = codecName[NDCODEC_LZ4HDF5];
+    output->compressedSize = compSize;
+
+    return output;
+}
+
+
+NDArray *decompressLZ4HDF5(NDArray *input, NDCodecStatus_t *status, char *errorMessage)
+{
+    // Sanity check
+    if (input->codec.name != codecName[NDCODEC_LZ4HDF5]) {
+        sprintf(errorMessage, "Invalid codec '%s', expected '%s'",
+                input->codec.name.c_str(), codecName[NDCODEC_LZ4HDF5].c_str());
+        *status = NDCODEC_ERROR;
+        return NULL;
+    }
+
+    NDArrayInfo_t info;
+    input->getInfo(&info);
+
+    NDArray *output = allocArray(input);
+
+    if (!output) {
+        sprintf(errorMessage, "Failed to allocate LZ4HDF5 output array");
+        *status = NDCODEC_ERROR;
+        return NULL;
+    }
+
+    int ret = decompress_lz4hdf5((const char*)input->pData, (char*)output->pData, info.totalBytes);
+
+    if (ret <= 0){
+        output->release();
+        sprintf(errorMessage, "Failed to LZ4HDF5 decompress");
+        *status = NDCODEC_ERROR;
+        return NULL;
+    }
+
+    output->codec.clear();
+
+    return output;
+}
+
 
 NDArray *compressBSLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage)
 {
@@ -714,6 +785,13 @@ void NDPluginCodec::processCallbacks(NDArray *pArray)
             break;
         }
 
+        case NDCODEC_LZ4HDF5: {
+            unlock();
+            result = compressLZ4HDF5(pArray, &codecStatus, errorMessage);
+            lock();
+            break;
+        }
+
         case NDCODEC_BSLZ4: {
             unlock();
             result = compressBSLZ4(pArray, &codecStatus, errorMessage);
@@ -750,6 +828,11 @@ void NDPluginCodec::processCallbacks(NDArray *pArray)
             result = decompressLZ4(pArray, &codecStatus, errorMessage);
             lock();
             setIntegerParam(NDCodecCompressor, NDCODEC_LZ4);
+        } else if (pArray->codec.name == codecName[NDCODEC_LZ4HDF5]) {
+            unlock();
+            result = decompressLZ4HDF5(pArray, &codecStatus, errorMessage);
+            lock();
+            setIntegerParam(NDCodecCompressor, NDCODEC_LZ4HDF5);
         } else if (pArray->codec.name == codecName[NDCODEC_BSLZ4]) {
             unlock();
             result = decompressBSLZ4(pArray, &codecStatus, errorMessage);

ADApp/pluginSrc/NDPluginCodec.h

@@ -33,6 +33,7 @@ typedef enum {
 typedef enum {
     NDCODEC_BLOSC_BLOSCLZ,
     NDCODEC_BLOSC_LZ4,
+    NDCODEC_BLOSC_LZ4HDF5,
     NDCODEC_BLOSC_LZ4HC,
     NDCODEC_BLOSC_SNAPPY,
     NDCODEC_BLOSC_ZLIB,
@@ -60,6 +61,8 @@ NDArray *compressBlosc(NDArray *input, int clevel, int shuffle, NDCodecBloscComp
 NDArray *decompressBlosc(NDArray *input, int numThreads, NDCodecStatus_t *status, char *errorMessage);
 NDArray *compressLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage);
 NDArray *decompressLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage);
+NDArray *compressLZ4HDF5(NDArray *input, NDCodecStatus_t *status, char *errorMessage);
+NDArray *decompressLZ4HDF5(NDArray *input, NDCodecStatus_t *status, char *errorMessage);
 NDArray *compressBSLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage);
 NDArray *decompressBSLZ4(NDArray *input, NDCodecStatus_t *status, char *errorMessage);