SDDS_binary.c File Reference

Detailed Description

SDDS binary data input and output routines.

This file contains the implementation of binary file handling functions for the SDDS (Self-Describing Data Sets) library. It includes functions for reading and writing binary data files in the SDDS format.

Copyright

• (c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
• (c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.

License

This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.

Author

M. Borland, C. Saunders, R. Soliday, H. Shang

Definition in file SDDS_binary.c.

#include "SDDS.h"
#include "SDDS_internal.h"
#include "mdb.h"
#include <string.h>
#include <errno.h>
#include <unistd.h>

Go to the source code of this file.

Functions
double	makeFloat64FromFloat80 (unsigned char x[16], int32_t byteOrder)
	Converts a 16-byte array representing a float80 value to a double.
int32_t	SDDS_SetBufferedRead (int32_t dummy)
int32_t	SDDS_SetDefaultIOBufferSize (int32_t newValue)
int32_t	SDDS_BufferedRead (void *target, int64_t targetSize, FILE *fp, SDDS_FILEBUFFER *fBuffer, int32_t type, int32_t byteOrder)
int32_t	SDDS_LZMABufferedRead (void *target, int64_t targetSize, struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer, int32_t type, int32_t byteOrder)
int32_t	SDDS_BufferedWrite (void *target, int64_t targetSize, FILE *fp, SDDS_FILEBUFFER *fBuffer)
int32_t	SDDS_LZMABufferedWrite (void *target, int64_t targetSize, struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer)
int32_t	SDDS_FlushBuffer (FILE *fp, SDDS_FILEBUFFER *fBuffer)
int32_t	SDDS_LZMAFlushBuffer (struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer)
int32_t	SDDS_WriteBinaryPage (SDDS_DATASET *SDDS_dataset)
int32_t	SDDS_UpdateBinaryPage (SDDS_DATASET *SDDS_dataset, uint32_t mode)
	Updates the binary page of an SDDS dataset.
int32_t	SDDS_fseek (FILE *fp, int64_t offset, int32_t dir)
	Sets the file position indicator for a given file stream with retry logic.
int32_t	SDDS_lzmaseek (struct lzmafile *lzmafp, int64_t offset, int32_t dir)
	Sets the file position indicator for a given LZMA file stream with retry logic.
int32_t	SDDS_WriteBinaryParameters (SDDS_DATASET *SDDS_dataset)
	Writes the binary parameters of the SDDS dataset.
int32_t	SDDS_WriteBinaryArrays (SDDS_DATASET *SDDS_dataset)
	Writes the binary arrays of the SDDS dataset to a file.
int32_t	SDDS_WriteBinaryColumns (SDDS_DATASET *SDDS_dataset)
	Writes the binary columns of an SDDS dataset to the associated file.
int32_t	SDDS_WriteNonNativeBinaryColumns (SDDS_DATASET *SDDS_dataset)
	Writes non-native endian binary columns of an SDDS dataset to the associated file.
int32_t	SDDS_WriteBinaryRow (SDDS_DATASET *SDDS_dataset, int64_t row)
	Writes a single binary row of an SDDS dataset to the associated file.
int32_t	SDDS_ReadRecoveryPossible (SDDS_DATASET *SDDS_dataset)
	Checks if any data in an SDDS page was recovered after an error was detected.
void	SDDS_SetReadRecoveryMode (SDDS_DATASET *SDDS_dataset, int32_t mode)
	Sets the read recovery mode for an SDDS dataset.
int32_t	SDDS_ReadBinaryPage (SDDS_DATASET *SDDS_dataset, int64_t sparse_interval, int64_t sparse_offset, int32_t sparse_statistics)
	Reads a binary page from an SDDS dataset.
int32_t	SDDS_ReadBinaryPageLastRows (SDDS_DATASET *SDDS_dataset, int64_t last_rows)
	Reads the last specified number of rows from a binary page of an SDDS dataset.
int32_t	SDDS_ReadBinaryPageDetailed (SDDS_DATASET *SDDS_dataset, int64_t sparse_interval, int64_t sparse_offset, int64_t last_rows, int32_t sparse_statistics)
	Reads a binary page from an SDDS dataset with detailed options.
int32_t	SDDS_WriteBinaryString (char *string, FILE *fp, SDDS_FILEBUFFER *fBuffer)
	Writes a binary string to a file with buffering.
int32_t	SDDS_LZMAWriteBinaryString (char *string, struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer)
	Writes a binary string to a file with LZMA compression.
char *	SDDS_ReadBinaryString (FILE *fp, SDDS_FILEBUFFER *fBuffer, int32_t skip)
	Reads a binary string from a file with buffering.
char *	SDDS_ReadLZMABinaryString (struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer, int32_t skip)
	Reads a binary string from an LZMA-compressed file with buffering.
int32_t	SDDS_ReadBinaryRow (SDDS_DATASET *SDDS_dataset, int64_t row, int32_t skip)
	Reads a binary row from the specified SDDS dataset.
int32_t	SDDS_ReadNewBinaryRows (SDDS_DATASET *SDDS_dataset)
	Reads new binary rows from the SDDS dataset.
int32_t	SDDS_ReadBinaryParameters (SDDS_DATASET *SDDS_dataset)
	Reads binary parameters from the specified SDDS dataset.
int32_t	SDDS_ReadBinaryArrays (SDDS_DATASET *SDDS_dataset)
	Reads binary arrays from an SDDS dataset.
int32_t	SDDS_ReadBinaryColumns (SDDS_DATASET *SDDS_dataset, int64_t sparse_interval, int64_t sparse_offset)
	Reads the binary columns from an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryColumns (SDDS_DATASET *SDDS_dataset)
	Reads the non-native endian binary columns from an SDDS dataset.
int32_t	SDDS_SwapEndsColumnData (SDDS_DATASET *SDDSin)
	Swaps the endianness of the column data in an SDDS dataset.
int32_t	SDDS_SwapEndsParameterData (SDDS_DATASET *SDDSin)
	Swaps the endianness of the parameter data in an SDDS dataset.
int32_t	SDDS_SwapEndsArrayData (SDDS_DATASET *SDDSin)
	Swaps the endianness of the array data in an SDDS dataset.
void	SDDS_SwapShort (short *data)
	Swaps the endianness of a short integer.
void	SDDS_SwapUShort (unsigned short *data)
	Swaps the endianness of an unsigned short integer.
void	SDDS_SwapLong (int32_t *data)
	Swaps the endianness of a 32-bit integer.
void	SDDS_SwapULong (uint32_t *data)
	Swaps the endianness of a 32-bit unsigned integer.
void	SDDS_SwapLong64 (int64_t *data)
	Swaps the endianness of a 64-bit integer.
void	SDDS_SwapULong64 (uint64_t *data)
	Swaps the endianness of a 64-bit unsigned integer.
void	SDDS_SwapFloat (float *data)
	Swaps the endianness of a float.
void	SDDS_SwapDouble (double *data)
	Swaps the endianness of a double.
void	SDDS_SwapLongDouble (long double *data)
	Swaps the endianness of a long double.
int32_t	SDDS_ReadNonNativePage (SDDS_DATASET *SDDS_dataset)
	Reads a non-native endian page from an SDDS dataset.
int32_t	SDDS_ReadNonNativePageSparse (SDDS_DATASET *SDDS_dataset, uint32_t mode, int64_t sparse_interval, int64_t sparse_offset)
	Reads a sparse non-native endian page from an SDDS dataset.
int32_t	SDDS_ReadNonNativePageDetailed (SDDS_DATASET *SDDS_dataset, uint32_t mode, int64_t sparse_interval, int64_t sparse_offset, int64_t last_rows)
	Reads a detailed non-native endian page from an SDDS dataset.
int32_t	SDDS_ReadNonNativePageLastRows (SDDS_DATASET *SDDS_dataset, int64_t last_rows)
	Reads the last few rows from a non-native endian page in an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryPage (SDDS_DATASET *SDDS_dataset, int64_t sparse_interval, int64_t sparse_offset)
	Reads a non-native endian binary page from an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryPageLastRows (SDDS_DATASET *SDDS_dataset, int64_t last_rows)
	Reads the last few rows from a non-native endian binary page in an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryPageDetailed (SDDS_DATASET *SDDS_dataset, int64_t sparse_interval, int64_t sparse_offset, int64_t last_rows)
	Reads a detailed non-native endian binary page from an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryParameters (SDDS_DATASET *SDDS_dataset)
	Reads non-native endian binary parameters from an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryArrays (SDDS_DATASET *SDDS_dataset)
	Reads non-native endian binary arrays from an SDDS dataset.
int32_t	SDDS_ReadNonNativeBinaryRow (SDDS_DATASET *SDDS_dataset, int64_t row, int32_t skip)
	Reads a non-native endian binary row from an SDDS dataset.
char *	SDDS_ReadNonNativeBinaryString (FILE *fp, SDDS_FILEBUFFER *fBuffer, int32_t skip)
	Reads a non-native endian binary string from a file.
char *	SDDS_ReadNonNativeLZMABinaryString (struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer, int32_t skip)
	Reads a non-native endian binary string from an LZMA-compressed file.
int32_t	SDDS_WriteNonNativeBinaryPage (SDDS_DATASET *SDDS_dataset)
	Writes a non-native endian binary page to an SDDS dataset.
int32_t	SDDS_WriteNonNativeBinaryParameters (SDDS_DATASET *SDDS_dataset)
	Writes non-native endian binary parameters to an SDDS dataset.
int32_t	SDDS_WriteNonNativeBinaryArrays (SDDS_DATASET *SDDS_dataset)
	Writes non-native endian binary arrays to an SDDS dataset.
int32_t	SDDS_WriteNonNativeBinaryRow (SDDS_DATASET *SDDS_dataset, int64_t row)
	Writes a non-native endian binary row to an SDDS dataset.
int32_t	SDDS_WriteNonNativeBinaryString (char *string, FILE *fp, SDDS_FILEBUFFER *fBuffer)
	Writes a non-native endian binary string to a file.
int32_t	SDDS_LZMAWriteNonNativeBinaryString (char *string, struct lzmafile *lzmafp, SDDS_FILEBUFFER *fBuffer)
	Writes a non-native endian binary string to an LZMA-compressed file.
int32_t	SDDS_UpdateNonNativeBinaryPage (SDDS_DATASET *SDDS_dataset, uint32_t mode)
	Updates a non-native endian binary page in an SDDS dataset.

Function Documentation

makeFloat64FromFloat80()

double makeFloat64FromFloat80	(	unsigned char	x[16],
		int32_t	byteOrder )

Converts a 16-byte array representing a float80 value to a double.

This function converts a 16-byte array, which represents an 80-bit floating-point (float80) value, to a standard double-precision (64-bit) floating-point value. The conversion handles different byte orders, supporting both big-endian and little-endian formats. On systems where long double is implemented as 64-bit (such as Windows and Mac), this function allows reading SDDS_LONGDOUBLE SDDS files with a loss of precision by translating float80 values to double.

Parameters

[in]	x	The 16-byte array representing the float80 value.
[in]	byteOrder	The byte order of the array, either SDDS_BIGENDIAN_SEEN or SDDS_LITTLEENDIAN_SEEN.

Returns

double The converted double-precision floating-point value.

Note

This function assumes that the input array x is correctly formatted as an 80-bit floating-point value. On systems where long double is 80 bits, the conversion preserves as much precision as possible within the limitations of the double-precision format. On systems with 64-bit long double, the function translates the value with inherent precision loss.

Definition at line 5892 of file SDDS_binary.c.

                                                                      {
  int exponent;
  uint64_t mantissa;
  unsigned char d[8] = {0};
  double result;
 
  if (byteOrder == SDDS_BIGENDIAN_SEEN) {
    /* conversion is done in little endian */
    char xx;
    int i;
    for (i = 0; i < 6; i++) {
      xx = x[0 + i];
      x[0 + i] = x[11 - i];
      x[11 - i] = xx;
    }
  }
 
  exponent = (((x[9] << 8) | x[8]) & 0x7FFF);
  mantissa =
    ((uint64_t)x[7] << 56) | ((uint64_t)x[6] << 48) | ((uint64_t)x[5] << 40) | ((uint64_t)x[4] << 32) |
    ((uint64_t)x[3] << 24) | ((uint64_t)x[2] << 16) | ((uint64_t)x[1] << 8) | (uint64_t)x[0];
 
  d[7] = x[9] & 0x80; /* Set sign. */
 
  if ((exponent == 0x7FFF) || (exponent == 0)) {
    /* Infinite, NaN or denormal */
    if (exponent == 0x7FFF) {
      /* Infinite or NaN */
      d[7] |= 0x7F;
      d[6] = 0xF0;
    } else {
      /* Otherwise it's denormal. It cannot be represented as double. Translate as singed zero. */
      memcpy(&result, d, 8);
      return result;
    }
  } else {
    /* Normal number. */
    exponent = exponent - 0x3FFF + 0x03FF; /*< exponent for double precision. */
 
    if (exponent <= -52) /*< Too small to represent. Translate as (signed) zero. */
    {
      memcpy(&result, d, 8);
      return result;
    } else if (exponent < 0) {
      /* Denormal, exponent bits are already zero here. */
    } else if (exponent >= 0x7FF) /*< Too large to represent. Translate as infinite. */
    {
      d[7] |= 0x7F;
      d[6] = 0xF0;
      memset(d, 0x00, 6);
      memcpy(&result, d, 8);
      return result;
    } else {
      /* Representable number */
      d[7] |= (exponent & 0x7F0) >> 4;
      d[6] |= (exponent & 0xF) << 4;
    }
  }
  /* Translate mantissa. */
 
  mantissa >>= 11;
 
  if (exponent < 0) {
    /* Denormal, further shifting is required here. */
    mantissa >>= (-exponent + 1);
  }
 
  d[0] = mantissa & 0xFF;
  d[1] = (mantissa >> 8) & 0xFF;
  d[2] = (mantissa >> 16) & 0xFF;
  d[3] = (mantissa >> 24) & 0xFF;
  d[4] = (mantissa >> 32) & 0xFF;
  d[5] = (mantissa >> 40) & 0xFF;
  d[6] |= (mantissa >> 48) & 0x0F;
 
  memcpy(&result, d, 8);
 
  if (byteOrder == SDDS_BIGENDIAN_SEEN) {
    /* convert back to big endian  */
    SDDS_SwapDouble(&result);
  }
 
  return result;
}

SDDS_BufferedRead()

int32_t SDDS_BufferedRead	(	void *	target,
		int64_t	targetSize,
		FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	type,
		int32_t	byteOrder )

Reads data from a file into a buffer, optimizing performance with buffering.

This function reads targetSize bytes from the file fp into the memory pointed to by target. It uses the provided fBuffer to buffer file data, improving read performance. If the data type is SDDS_LONGDOUBLE and the long double precision is not 18 digits, it handles conversion to double precision if the environment variable SDDS_LONGDOUBLE_64BITS is not set.

If target is NULL, the function skips over targetSize bytes in the file.

Parameters

target	Pointer to the memory location where the data will be stored. If NULL, the data is skipped.
targetSize	The number of bytes to read from the file.
fp	The file pointer from which data is read.
fBuffer	Pointer to an SDDS_FILEBUFFER structure used for buffering file data.
type	The SDDS data type of the data being read (e.g., SDDS_LONGDOUBLE).
byteOrder	The byte order of the data (SDDS_LITTLEENDIAN or SDDS_BIGENDIAN).

Returns

Returns 1 on success; returns 0 on error.

Definition at line 96 of file SDDS_binary.c.

                                                                                                                                 {
  int float80tofloat64 = 0;
  if ((LDBL_DIG != 18) && (type == SDDS_LONGDOUBLE)) {
    if (getenv("SDDS_LONGDOUBLE_64BITS") == NULL) {
      targetSize *= 2;
      float80tofloat64 = 1;
    }
  }
  if (!fBuffer->bufferSize) {
    /* just read into users buffer or seek if no buffer given */
    if (!target)
      return !fseek(fp, (long)targetSize, SEEK_CUR);
    else {
      if (float80tofloat64) {
        unsigned char x[16];
        double d;
        int64_t shift = 0;
        while (shift < targetSize) {
          if (fread(&x, (size_t)1, 16, fp) != 16)
            return 0;
          d = makeFloat64FromFloat80(x, byteOrder);
          memcpy((char *)target + shift, &d, 8);
          shift += 16;
        }
        return 1;
      } else {
        return fread(target, (size_t)1, (size_t)targetSize, fp) == targetSize;
      }
    }
  }
  if ((fBuffer->bytesLeft -= targetSize) >= 0) {
    /* sufficient data is already in the buffer */
    if (target) {
      if (float80tofloat64) {
        unsigned char x[16];
        double d;
        int64_t shift = 0;
        while (shift < targetSize) {
          memcpy(x, (char *)fBuffer->data + shift, 16);
          d = makeFloat64FromFloat80(x, byteOrder);
          memcpy((char *)target + shift, &d, 8);
          shift += 16;
        }
      } else {
        memcpy((char *)target, (char *)fBuffer->data, targetSize);
      }
    }
    fBuffer->data += targetSize;
    return 1;
  } else {
    /* need to read additional data into buffer */
    int64_t bytesNeeded, offset;
    fBuffer->bytesLeft += targetSize; /* adds back amount subtracted above */
 
    /* first, use the data that is already available. this cleans out the buffer */
    if ((offset = fBuffer->bytesLeft)) {
      /* some data is available in the buffer */
      if (target) {
        if (float80tofloat64) {
          unsigned char x[16];
          double d;
          int64_t shift = 0;
          while (shift < offset) {
            memcpy(x, (char *)fBuffer->data + shift, 16);
            d = makeFloat64FromFloat80(x, byteOrder);
            memcpy((char *)target + shift, &d, 8);
            shift += 16;
          }
        } else {
          memcpy((char *)target, (char *)fBuffer->data, offset);
        }
      }
      bytesNeeded = targetSize - offset;
      fBuffer->bytesLeft = 0;
    } else {
      bytesNeeded = targetSize;
    }
    fBuffer->data = fBuffer->buffer;
 
    if (fBuffer->bufferSize < bytesNeeded) {
      /* just read what is needed directly into user's memory or seek */
      if (!target)
        return !fseek(fp, (long)bytesNeeded, SEEK_CUR);
      else {
        if (float80tofloat64) {
          unsigned char x[16];
          double d;
          int64_t shift = 0;
          while (shift < bytesNeeded) {
            if (fread(&x, (size_t)1, 16, fp) != 16)
              return 0;
            d = makeFloat64FromFloat80(x, byteOrder);
            memcpy((char *)target + offset + shift, &d, 8);
            shift += 16;
          }
          return 1;
        } else {
          return fread((char *)target + offset, (size_t)1, (size_t)bytesNeeded, fp) == bytesNeeded;
        }
      }
    }
 
    /* fill the buffer */
    if ((fBuffer->bytesLeft = fread(fBuffer->data, (size_t)1, (size_t)fBuffer->bufferSize, fp)) < bytesNeeded)
      return 0;
    if (target) {
      if (float80tofloat64) {
        unsigned char x[16];
        double d;
        int64_t shift = 0;
        while (shift < bytesNeeded) {
          memcpy(x, (char *)fBuffer->data + shift, 16);
          d = makeFloat64FromFloat80(x, byteOrder);
          memcpy((char *)target + offset + shift, &d, 8);
          shift += 16;
        }
      } else {
        memcpy((char *)target + offset, (char *)fBuffer->data, bytesNeeded);
      }
    }
    fBuffer->data += bytesNeeded;
    fBuffer->bytesLeft -= bytesNeeded;
    return 1;
  }
}

SDDS_BufferedWrite()

int32_t SDDS_BufferedWrite	(	void *	target,
		int64_t	targetSize,
		FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer )

Writes data to a file using a buffer to optimize performance.

This function writes targetSize bytes from the memory pointed to by target to the file fp. It uses the provided fBuffer to buffer file data, improving write performance. If the buffer is full, it flushes the buffer to the file before writing more data.

Parameters

target	Pointer to the memory location of the data to write.
targetSize	The number of bytes to write to the file.
fp	The file pointer to which data is written.
fBuffer	Pointer to an SDDS_FILEBUFFER structure used for buffering file data.

Returns

Returns 1 on success; returns 0 on error.

Definition at line 484 of file SDDS_binary.c.

                                                                                                 {
  if (!fBuffer->bufferSize) {
    return fwrite(target, (size_t)1, (size_t)targetSize, fp) == targetSize;
  }
  if ((fBuffer->bytesLeft -= targetSize) >= 0) {
    memcpy((char *)fBuffer->data, (char *)target, targetSize);
    fBuffer->data += targetSize;
#ifdef DEBUG
    fprintf(stderr, "SDDS_BufferedWrite of %" PRId64 " bytes done in-memory, %" PRId64 " bytes left\n", targetSize, fBuffer->bytesLeft);
#endif
    return 1;
  } else {
    int64_t lastLeft;
    /* add back what was subtracted in test above.
     * lastLeft is the number of bytes left in the buffer before doing anything
     * and also the number of bytes from the users data that get copied into the buffer.
     */
    lastLeft = (fBuffer->bytesLeft += targetSize);
    /* copy part of the data into the buffer and write the buffer out */
    memcpy((char *)fBuffer->data, (char *)target, (size_t)fBuffer->bytesLeft);
    if (fwrite(fBuffer->buffer, (size_t)1, (size_t)fBuffer->bufferSize, fp) != fBuffer->bufferSize)
      return 0;
    if (fflush(fp)) {
      SDDS_SetError("Problem flushing file (SDDS_BufferedWrite)");
      SDDS_SetError(strerror(errno));
      return 0;
    }
    /* reset the data pointer and the bytesLeft value.
     * also, determine if the remaining data is too large for the buffer.
     * if so, just write it out.
     */
    fBuffer->data = fBuffer->buffer;
    if ((targetSize -= lastLeft) > (fBuffer->bytesLeft = fBuffer->bufferSize)) {
      return fwrite((char *)target + lastLeft, (size_t)1, (size_t)targetSize, fp) == targetSize;
    }
    /* copy remaining data into the buffer.
     * could do this with a recursive call, but this is more efficient.
     */
    memcpy((char *)fBuffer->data, (char *)target + lastLeft, targetSize);
    fBuffer->data += targetSize;
    fBuffer->bytesLeft -= targetSize;
    return 1;
  }
}

SDDS_FlushBuffer()

int32_t SDDS_FlushBuffer	(	FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer )

Flushes the buffered data to a file to ensure all data is written.

This function writes any remaining data in the buffer (fBuffer) to the file pointed to by fp. If the buffer contains data, it writes the data to the file, resets the buffer, and flushes the file's output buffer using fflush. This ensures that all buffered data is physically written to the file.

Parameters

fp	The file pointer to which buffered data will be written.
fBuffer	Pointer to an SDDS_FILEBUFFER structure containing the buffered data.

Returns

Returns 1 on success; returns 0 on error.

Note

If fBuffer->bufferSize is zero, the function will only call fflush(fp).

Warning

If fp or fBuffer is NULL, the function sets an error message and returns 0.

Definition at line 632 of file SDDS_binary.c.

                                                             {
  int64_t writeBytes;
  if (!fp) {
    SDDS_SetError("Unable to flush buffer: file pointer is NULL. (SDDS_FlushBuffer)");
    return 0;
  }
  if (!fBuffer->bufferSize) {
    if (fflush(fp)) {
      SDDS_SetError("Problem flushing file (SDDS_FlushBuffer.1)");
      SDDS_SetError(strerror(errno));
      return 0;
    }
    return 1;
  }
  if (!fBuffer) {
    SDDS_SetError("Unable to flush buffer: buffer pointer is NULL. (SDDS_FlushBuffer)");
    return 0;
  }
  if ((writeBytes = fBuffer->bufferSize - fBuffer->bytesLeft)) {
    if (writeBytes < 0) {
      SDDS_SetError("Unable to flush buffer: negative byte count (SDDS_FlushBuffer).");
      return 0;
    }
#ifdef DEBUG
    fprintf(stderr, "Writing %" PRId64 " bytes to disk\n", writeBytes);
#endif
    if (fwrite(fBuffer->buffer, 1, writeBytes, fp) != writeBytes) {
      SDDS_SetError("Unable to flush buffer: write operation failed (SDDS_FlushBuffer).");
      return 0;
    }
    fBuffer->bytesLeft = fBuffer->bufferSize;
    fBuffer->data = fBuffer->buffer;
  }
  if (fflush(fp)) {
    SDDS_SetError("Problem flushing file (SDDS_FlushBuffer.2)");
    SDDS_SetError(strerror(errno));
    return 0;
  }
  return 1;
}

SDDS_fseek()

int32_t SDDS_fseek	(	FILE *	fp,
		int64_t	offset,
		int32_t	dir )

Sets the file position indicator for a given file stream with retry logic.

Attempts to set the file position indicator for the specified file stream (fp) to a new position defined by offset and dir. The function retries the fseek operation up to FSEEK_TRIES times in case of transient failures, implementing a delay between attempts.

Parameters

fp	Pointer to the FILE stream whose position indicator is to be set.
offset	Number of bytes to offset from the position specified by dir.
dir	Positioning directive, which can be one of: SEEK_SET to set the position relative to the beginning of the file, SEEK_CUR to set the position relative to the current position, SEEK_END to set the position relative to the end of the file.

Returns

• Returns 0 if the operation is successful.
• Returns -1 if all retry attempts fail to set the file position.

The function attempts to set the file position using fseek. If fseek fails, it sleeps for 1 second (or 1 second using nanosleep on vxWorks systems) before retrying. After FSEEK_TRIES unsuccessful attempts, it reports a warning and returns -1.

Note

• The function is designed to handle temporary file access issues by retrying the fseek operation.
• It is not suitable for non-recoverable fseek errors, which will cause it to fail after retries.

Definition at line 1258 of file SDDS_binary.c.

                                                          {
  int32_t try;
#if defined(vxWorks)
  struct timespec rqtp;
  rqtp.tv_sec = 1;
  rqtp.tv_nsec = 0;
#endif
  for (try = 0; try < FSEEK_TRIES; try++) {
    if (fseek(fp, offset, dir) == -1) {
#if defined(vxWorks)
      nanosleep(&rqtp, NULL);
#else
      sleep(1);
#endif
    } else
      break;
  }
  if (try == 0)
    return 0;
  if (try == FSEEK_TRIES) {
    fputs("warning: fseek problems--unable to recover\n", stderr);
    return -1;
  }
  fputs("warning: fseek problems--recovered\n", stderr);
  return 0;
}

SDDS_LZMABufferedRead()

int32_t SDDS_LZMABufferedRead	(	void *	target,
		int64_t	targetSize,
		struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	type,
		int32_t	byteOrder )

Reads data from an LZMA-compressed file into a buffer, optimizing performance with buffering.

This function reads targetSize bytes from the LZMA-compressed file lzmafp into the memory pointed to by target. It uses the provided fBuffer to buffer file data, improving read performance. If the data type is SDDS_LONGDOUBLE and the long double precision is not 18 digits, it handles conversion to double precision if the environment variable SDDS_LONGDOUBLE_64BITS is not set.

If target is NULL, the function skips over targetSize bytes in the file.

Parameters

target	Pointer to the memory location where the data will be stored. If NULL, the data is skipped.
targetSize	The number of bytes to read from the file.
lzmafp	The LZMA file pointer from which data is read.
fBuffer	Pointer to an SDDS_FILEBUFFER structure used for buffering file data.
type	The SDDS data type of the data being read (e.g., SDDS_LONGDOUBLE).
byteOrder	The byte order of the data (SDDS_LITTLEENDIAN or SDDS_BIGENDIAN).

Returns

Returns 1 on success; returns 0 on error.

Note

This function requires that fBuffer->bufferSize is non-zero. If it is zero, an error is set.

Definition at line 243 of file SDDS_binary.c.

                                                                                                                                                    {
  int float80tofloat64 = 0;
  if (!fBuffer->bufferSize) {
    SDDS_SetError("You must presently have a nonzero file buffer to use LZMA (reading/writing .lzma or .xz files)");
    return 0;
  }
  if ((LDBL_DIG != 18) && (type == SDDS_LONGDOUBLE)) {
    if (getenv("SDDS_LONGDOUBLE_64BITS") == NULL) {
      targetSize *= 2;
      float80tofloat64 = 1;
    }
  }
  if ((fBuffer->bytesLeft -= targetSize) >= 0) {
    if (target) {
      if (float80tofloat64) {
        unsigned char x[16];
        double d;
        int64_t shift = 0;
        while (shift < targetSize) {
          memcpy(x, (char *)fBuffer->data + shift, 16);
          d = makeFloat64FromFloat80(x, byteOrder);
          memcpy((char *)target + shift, &d, 8);
          shift += 16;
        }
      } else {
        memcpy((char *)target, (char *)fBuffer->data, targetSize);
      }
    }
    fBuffer->data += targetSize;
    return 1;
  } else {
    int64_t bytesNeeded, offset;
    fBuffer->bytesLeft += targetSize;
    if ((offset = fBuffer->bytesLeft)) {
      if (target) {
        if (float80tofloat64) {
          unsigned char x[16];
          double d;
          int64_t shift = 0;
          while (shift < offset) {
            memcpy(x, (char *)fBuffer->data + shift, 16);
            d = makeFloat64FromFloat80(x, byteOrder);
            memcpy((char *)target + shift, &d, 8);
            shift += 16;
          }
        } else {
          memcpy((char *)target, (char *)fBuffer->data, offset);
        }
      }
      bytesNeeded = targetSize - offset;
      fBuffer->bytesLeft = 0;
    } else {
      bytesNeeded = targetSize;
    }
    fBuffer->data = fBuffer->buffer;
 
    if (fBuffer->bufferSize < bytesNeeded) {
      /* just read what is needed directly into user's memory or seek */
      if (!target)
        return !lzma_seek(lzmafp, (long)bytesNeeded, SEEK_CUR);
      else {
        if (float80tofloat64) {
          unsigned char x[16];
          double d;
          int64_t shift = 0;
          while (shift < bytesNeeded) {
            if (lzma_read(lzmafp, &x, 16) != 16)
              return 0;
            d = makeFloat64FromFloat80(x, byteOrder);
            memcpy((char *)target + offset + shift, &d, 8);
            shift += 16;
          }
          return 1;
        } else {
          return lzma_read(lzmafp, (char *)target + offset, (size_t)bytesNeeded) == bytesNeeded;
        }
      }
    }
 
    if ((fBuffer->bytesLeft = lzma_read(lzmafp, fBuffer->data, (size_t)fBuffer->bufferSize)) < bytesNeeded)
      return 0;
    if (target) {
      if (float80tofloat64) {
        unsigned char x[16];
        double d;
        int64_t shift = 0;
        while (shift < bytesNeeded) {
          memcpy(x, (char *)fBuffer->data + shift, 16);
          d = makeFloat64FromFloat80(x, byteOrder);
          memcpy((char *)target + offset + shift, &d, 8);
          shift += 16;
        }
      } else {
        memcpy((char *)target + offset, (char *)fBuffer->data, bytesNeeded);
      }
    }
    fBuffer->data += bytesNeeded;
    fBuffer->bytesLeft -= bytesNeeded;
    return 1;
  }
}

SDDS_LZMABufferedWrite()

int32_t SDDS_LZMABufferedWrite	(	void *	target,
		int64_t	targetSize,
		struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer )

Writes data to an LZMA-compressed file using a buffer to optimize performance.

This function writes targetSize bytes from the memory pointed to by target to the LZMA-compressed file referenced by lzmafp. It uses the provided fBuffer to buffer data before writing to the file, which can improve write performance by reducing the number of write operations.

If there is enough space in the buffer (fBuffer), the data is copied into the buffer. If the buffer does not have enough space to hold the data, the buffer is flushed to the file, and the function recursively calls itself to handle the remaining data.

Parameters

target	Pointer to the memory location of the data to write.
targetSize	The number of bytes to write to the file.
lzmafp	The LZMA file pointer to which data is written.
fBuffer	Pointer to an SDDS_FILEBUFFER structure used for buffering data.

Returns

Returns 1 on success; returns 0 on error.

Note

This function requires that fBuffer->bufferSize is non-zero. If it is zero, the function sets an error message and returns 0.

Definition at line 550 of file SDDS_binary.c.

                                                                                                                    {
  if (!fBuffer->bufferSize) {
    SDDS_SetError("You must presently have a nonzero file buffer to use lzma (reading/writing .xz files)");
    return 0;
  }
  if ((fBuffer->bytesLeft -= targetSize) >= 0) {
    memcpy((char *)fBuffer->data, (char *)target, targetSize);
    fBuffer->data += targetSize;
    return 1;
  } else {
    int64_t lastLeft;
    lastLeft = (fBuffer->bytesLeft += targetSize);
    memcpy((char *)fBuffer->data, (char *)target, (size_t)fBuffer->bytesLeft);
    if (lzma_write(lzmafp, fBuffer->buffer, (size_t)fBuffer->bufferSize) != fBuffer->bufferSize)
      return 0;
    fBuffer->bytesLeft = fBuffer->bufferSize;
    fBuffer->data = fBuffer->buffer;
    return SDDS_LZMABufferedWrite((char *)target + lastLeft, targetSize - lastLeft, lzmafp, fBuffer);
  }
}

SDDS_LZMAFlushBuffer()

int32_t SDDS_LZMAFlushBuffer	(	struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer )

Flushes the buffered data to an LZMA-compressed file to ensure all data is written.

This function writes any remaining data in the buffer (fBuffer) to the LZMA-compressed file pointed to by lzmafp. If the buffer contains data, it writes the data to the file, resets the buffer, ensuring that all buffered data is physically written to the file.

Parameters

lzmafp	The LZMA file pointer to which buffered data will be written.
fBuffer	Pointer to an SDDS_FILEBUFFER structure containing the buffered data.

Returns

Returns 1 on success; returns 0 on error.

Note

This function assumes that fBuffer->bufferSize is non-zero and fBuffer is properly initialized.

Definition at line 687 of file SDDS_binary.c.

                                                                                {
  int32_t writeBytes;
  if ((writeBytes = fBuffer->bufferSize - fBuffer->bytesLeft)) {
    if (lzma_write(lzmafp, fBuffer->buffer, writeBytes) != writeBytes)
      return 0;
    fBuffer->bytesLeft = fBuffer->bufferSize;
    fBuffer->data = fBuffer->buffer;
  }
  return 1;
}

SDDS_lzmaseek()

int32_t SDDS_lzmaseek	(	struct lzmafile *	lzmafp,
		int64_t	offset,
		int32_t	dir )

Sets the file position indicator for a given LZMA file stream with retry logic.

Attempts to set the file position indicator for the specified LZMA file stream (lzmafp) to a new position defined by offset and dir. The function retries the lzma_seek operation up to FSEEK_TRIES times in case of transient failures, implementing a delay between attempts.

Parameters

lzmafp	Pointer to the lzmafile stream whose position indicator is to be set.
offset	Number of bytes to offset from the position specified by dir.
dir	Positioning directive, which can be one of: SEEK_SET to set the position relative to the beginning of the file, SEEK_CUR to set the position relative to the current position, SEEK_END to set the position relative to the end of the file.

Returns

• Returns 0 if the operation is successful.
• Returns -1 if all retry attempts fail to set the file position.

The function attempts to set the file position using lzma_seek. If lzma_seek fails, it sleeps for 1 second (or 1 second using nanosleep on vxWorks systems) before retrying. After FSEEK_TRIES unsuccessful attempts, it reports a warning and returns -1.

Note

• The function is designed to handle temporary file access issues by retrying the lzma_seek operation.
• It is not suitable for non-recoverable lzma_seek errors, which will cause it to fail after retries.

Definition at line 1312 of file SDDS_binary.c.

                                                                            {
  int32_t try;
#if defined(vxWorks)
  struct timespec rqtp;
  rqtp.tv_sec = 1;
  rqtp.tv_nsec = 0;
#endif
  for (try = 0; try < FSEEK_TRIES; try++) {
    if (lzma_seek(lzmafp, offset, dir) == -1) {
#if defined(vxWorks)
      nanosleep(&rqtp, NULL);
#else
      sleep(1);
#endif
    } else
      break;
  }
  if (try == 0)
    return 0;
  if (try == FSEEK_TRIES) {
    fputs("warning: lzma_seek problems--unable to recover\n", stderr);
    return -1;
  }
  fputs("warning: lzma_seek problems--recovered\n", stderr);
  return 0;
}

SDDS_LZMAWriteBinaryString()

int32_t SDDS_LZMAWriteBinaryString	(	char *	string,
		struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer )

Writes a binary string to a file with LZMA compression.

This function writes a binary string to the specified LZMA-compressed file by first writing the length of the string followed by the string's content. If the input string is NULL, an empty string is written instead. The writing operation utilizes LZMA buffered write functions to ensure data is compressed appropriately.

Parameters

[in]	string	The null-terminated string to be written. If NULL, an empty string is written.
[in]	lzmafp	The LZMA file pointer to write to. Must be a valid, open LZMA-compressed file in write mode.
[in,out]	fBuffer	Pointer to the file buffer used for buffered writing operations.

Returns

int32_t Returns 1 on success, 0 on failure.

Return values

1	Operation was successful.
0	An error occurred during writing.

Definition at line 2549 of file SDDS_binary.c.

                                                                                                    {
  int32_t length;
  static char *dummy_string = "";
  if (!string)
    string = dummy_string;
  length = strlen(string);
  if (!SDDS_LZMABufferedWrite(&length, sizeof(length), lzmafp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing length");
    return (0);
  }
  if (length && !SDDS_LZMABufferedWrite(string, sizeof(*string) * length, lzmafp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing contents");
    return (0);
  }
  return (1);
}

SDDS_LZMAWriteNonNativeBinaryString()

int32_t SDDS_LZMAWriteNonNativeBinaryString	(	char *	string,
		struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer )

Writes a non-native endian binary string to an LZMA-compressed file.

This function writes a binary string to the specified LZMA-compressed file pointer, handling non-native endianness. It first writes the length of the string as a 32-bit integer with byte order swapped. If the string is not to be skipped, it then writes the string data itself followed by a null terminator. If the input string is NULL, an empty string is written instead.

Parameters

[in]	string	The string to write. If NULL, an empty string is written.
[in]	lzmafp	Pointer to the LZMAFILE where the string will be written.
[in]	fBuffer	Pointer to the SDDS_FILEBUFFER structure used for buffered writing.

Returns

int32_t Returns 1 on successful writing of the string, or 0 if an error occurred.

Return values

1	The string was successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures or memory allocation issues.

Note

The caller is responsible for ensuring that the LZMAFILE pointer lzmafp is valid and open for writing. This function does not perform memory allocation for the string; it assumes that the string is already allocated and managed appropriately.

Definition at line 5636 of file SDDS_binary.c.

                                                                                                             {
  int32_t length;
  static char *dummy_string = "";
  if (!string)
    string = dummy_string;
  length = strlen(string);
  SDDS_SwapLong(&length);
  if (!SDDS_LZMABufferedWrite(&length, sizeof(length), lzmafp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing length");
    return (0);
  }
  SDDS_SwapLong(&length);
  if (length && !SDDS_LZMABufferedWrite(string, sizeof(*string) * length, lzmafp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing contents");
    return (0);
  }
  return (1);
}

SDDS_ReadBinaryArrays()

int32_t SDDS_ReadBinaryArrays

(

SDDS_DATASET *

SDDS_dataset

)

Reads binary arrays from an SDDS dataset.

This function iterates through all array definitions within the specified SDDS dataset and reads their binary data from the underlying file. It handles various compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. For each array, the function reads its definition, dimensions, and data elements, allocating and managing memory as necessary. String arrays are handled by reading each string individually, while other data types are read in bulk.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successful reading of all arrays, or 0 if an error occurred.

Return values

1	All arrays were successfully read and stored.
0	An error occurred during the read operation, such as I/O failures, memory allocation issues, or corrupted array definitions.

Note

The caller is responsible for ensuring that the SDDS_dataset structure is properly initialized and that memory allocations for arrays are managed appropriately to prevent memory leaks.

Definition at line 3059 of file SDDS_binary.c.

                                                          {
  int32_t i, j;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  /*  static char buffer[SDDS_MAXLINE]; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_ARRAY *array;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadBinaryArrays"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_arrays)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  if (!SDDS_dataset->array) {
    SDDS_SetError("Unable to read array--pointer to structure storage area is NULL (SDDS_ReadBinaryArrays)");
    return (0);
  }
  for (i = 0; i < layout->n_arrays; i++) {
    array = SDDS_dataset->array + i;
    if (array->definition && !SDDS_FreeArrayDefinition(array->definition)) {
      SDDS_SetError("Unable to get array--array definition corrupted (SDDS_ReadBinaryArrays)");
      return (0);
    }
    if (!SDDS_CopyArrayDefinition(&array->definition, layout->array_definition + i)) {
      SDDS_SetError("Unable to read array--definition copy failed (SDDS_ReadBinaryArrays)");
      return (0);
    }
    /*if (array->dimension) free(array->dimension); */
    if (!(array->dimension = SDDS_Realloc(array->dimension, sizeof(*array->dimension) * array->definition->dimensions))) {
      SDDS_SetError("Unable to read array--allocation failure (SDDS_ReadBinaryArrays)");
      return (0);
    }
#if defined(zLib)
    if (SDDS_dataset->layout.gzipFile) {
      if (!SDDS_GZipBufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, gzfp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadBinaryArrays)");
        return (0);
      }
    } else {
#endif
      if (SDDS_dataset->layout.lzmaFile) {
        if (!SDDS_LZMABufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, lzmafp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadBinaryArrays)");
          return (0);
        }
      } else {
        if (!SDDS_BufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, fp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadBinaryArrays)");
          return (0);
        }
      }
#if defined(zLib)
    }
#endif
    array->elements = 1;
    for (j = 0; j < array->definition->dimensions; j++)
      array->elements *= array->dimension[j];
    if (array->data)
      free(array->data);
    array->data = array->pointer = NULL;
    if (array->elements == 0)
      continue;
    if (array->elements < 0) {
      SDDS_SetError("Unable to read array--number of elements is negative (SDDS_ReadBinaryArrays)");
      return (0);
    }
    if (!(array->data = SDDS_Realloc(array->data, array->elements * SDDS_type_size[array->definition->type - 1]))) {
      SDDS_SetError("Unable to read array--allocation failure (SDDS_ReadBinaryArrays)");
      return (0);
    }
    if (array->definition->type == SDDS_STRING) {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        for (j = 0; j < array->elements; j++) {
          if (!(((char **)(array->data))[j] = SDDS_ReadGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadBinaryArrays)");
            return (0);
          }
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          for (j = 0; j < array->elements; j++) {
            if (!(((char **)(array->data))[j] = SDDS_ReadLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadBinaryArrays)");
              return (0);
            }
          }
        } else {
          for (j = 0; j < array->elements; j++) {
            if (!(((char **)(array->data))[j] = SDDS_ReadBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadBinaryArrays)");
              return (0);
            }
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, gzfp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadBinaryArrays)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, lzmafp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadBinaryArrays)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, fp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadBinaryArrays)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
  return (1);
}

SDDS_ReadBinaryColumns()

int32_t SDDS_ReadBinaryColumns	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	sparse_interval,
		int64_t	sparse_offset )

Reads the binary columns from an SDDS dataset.

This function iterates through all column definitions within the specified SDDS dataset and reads their binary data from the underlying file. It handles various compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. For each column, the function reads data for each row, managing memory allocation for string columns as necessary. Non-string data types are read in bulk for each column.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successful reading of all columns, or 0 if an error occurred.

Return values

1	All columns were successfully read and stored.
0	An error occurred during the read operation, such as I/O failures, memory allocation issues, or corrupted column definitions.

Note

The caller is responsible for ensuring that the SDDS_dataset structure is properly initialized and that memory allocations for columns are managed appropriately to prevent memory leaks.

Definition at line 3224 of file SDDS_binary.c.

                                                                                                           {
  int64_t i, j, k, row;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  /*  static char buffer[SDDS_MAXLINE]; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadBinaryColumns"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_columns || !SDDS_dataset->n_rows)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
 
  for (i = 0; i < layout->n_columns; i++) {
    if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
      continue;
    if (layout->column_definition[i].type == SDDS_STRING) {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        for (row = 0; row < SDDS_dataset->n_rows; row++) {
          if (((char ***)SDDS_dataset->data)[i][row])
            free((((char ***)SDDS_dataset->data)[i][row]));
          if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadBinaryColumns)");
            return (0);
          }
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadBinaryColumms)");
              return (0);
            }
          }
        } else {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadBinaryColumms)");
              return (0);
            }
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, gzfp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadBinaryColumns)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, lzmafp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadBinaryColumns)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, fp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadBinaryColumns)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
 
  if (sparse_interval == 1 && sparse_offset == 0) {
    return(1);
  }
 
  j = SDDS_dataset->n_rows;
  for (i = 0; i < layout->n_columns; i++) {
    j = k = 0;
    switch (layout->column_definition[i].type) {
    case SDDS_SHORT:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((short*)SDDS_dataset->data[i])[j] = ((short*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_USHORT:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((unsigned short*)SDDS_dataset->data[i])[j] = ((unsigned short*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_LONG:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((int32_t*)SDDS_dataset->data[i])[j] = ((int32_t*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_ULONG:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((uint32_t*)SDDS_dataset->data[i])[j] = ((uint32_t*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_LONG64:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((int64_t*)SDDS_dataset->data[i])[j] = ((int64_t*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_ULONG64:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((uint64_t*)SDDS_dataset->data[i])[j] = ((uint64_t*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_FLOAT:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((float*)SDDS_dataset->data[i])[j] = ((float*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_DOUBLE:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((double*)SDDS_dataset->data[i])[j] = ((double*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_LONGDOUBLE:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((long double*)SDDS_dataset->data[i])[j] = ((long double*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    case SDDS_STRING:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((char**)SDDS_dataset->data[i])[j] = ((char**)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      for (k=j; k<SDDS_dataset->n_rows; k++) {
        if (((char ***)SDDS_dataset->data)[i][k]) {
          free((((char ***)SDDS_dataset->data)[i][k]));
          ((char ***)SDDS_dataset->data)[i][k] = NULL;
        }
      }
 
      break; 
    case SDDS_CHARACTER:
      for (row = sparse_offset; row < SDDS_dataset->n_rows; row++) {
        if (k % sparse_interval == 0) {
          ((char*)SDDS_dataset->data[i])[j] = ((char*)SDDS_dataset->data[i])[row];
          j++;
        }
        k++;
      }
      break; 
    default:
      break;
    }
  }
    
  SDDS_dataset->n_rows = j;
 
  return (1);
}

SDDS_ReadBinaryPage()

int32_t SDDS_ReadBinaryPage	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	sparse_interval,
		int64_t	sparse_offset,
		int32_t	sparse_statistics )

Reads a binary page from an SDDS dataset.

This function reads a binary page from the specified SDDS dataset. It allows for sparse reading by specifying the sparse_interval and sparse_offset parameters, enabling the reading of data at specified intervals or starting from a specific offset.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
sparse_interval	Interval at which to read rows. A value greater than 1 enables sparse reading.
sparse_offset	Number of initial rows to skip before starting to read data.
sparse_statistics	Flag indicating whether to compute statistics during sparse reading: 0: No statistics. 1: Compute average. 2: Compute median. 3: Compute minimum. 4: Compute maximum.

Returns

• Returns the page number on successful read.
• Returns -1 if the end-of-file is reached.
• Returns 0 on error.

The function internally calls SDDS_ReadBinaryPageDetailed with the provided parameters to perform the actual reading. It handles various scenarios, including non-native byte orders and different data layouts (row-major or column-major).

Note

• This function is typically called to read data pages in bulk, allowing for efficient data access by skipping unnecessary rows.
• Sparse statistics can be used to reduce the amount of data by computing aggregated values.
• The function assumes that the dataset has been properly initialized and that the file pointers are correctly set up.

Definition at line 2120 of file SDDS_binary.c.

                                                                                                                                   {
  return SDDS_ReadBinaryPageDetailed(SDDS_dataset, sparse_interval, sparse_offset, 0, sparse_statistics);
}

SDDS_ReadBinaryPageDetailed()

int32_t SDDS_ReadBinaryPageDetailed	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	sparse_interval,
		int64_t	sparse_offset,
		int64_t	last_rows,
		int32_t	sparse_statistics )

Reads a binary page from an SDDS dataset with detailed options.

This function reads a binary page from the specified SDDS dataset, providing detailed control over the reading process. It supports sparse reading, reading a specific number of rows from the end, and computing statistics on the data.

Typically, this function is not called directly. Instead, it is invoked through higher-level functions such as SDDS_ReadBinaryPage or SDDS_ReadBinaryPageLastRows, which provide simplified interfaces for common reading scenarios.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
sparse_interval	Interval at which to read rows. A value greater than 1 enables sparse reading.
sparse_offset	Number of initial rows to skip before starting to read data.
last_rows	The number of rows to read from the end of the dataset. If 0, all rows are read.
sparse_statistics	Flag indicating whether to compute statistics during sparse reading: 0: No statistics. 1: Compute average. 2: Compute median. 3: Compute minimum. 4: Compute maximum.

Returns

• Returns the page number on successful read.
• Returns -1 if the end-of-file is reached.
• Returns 0 on error.

The function performs the following steps:

• Checks if the dataset has been auto-recovered; if so, it returns -1.
• Determines if the dataset uses native or non-native byte order and delegates to SDDS_ReadNonNativePageDetailed if necessary.
• Initializes file pointers based on the compression format (standard, gzip, LZMA).
• Allocates and initializes the buffer for reading if not already allocated.
• Reads the number of rows from the binary file, handling both 32-bit and 64-bit row counts.
• Validates the row count and ensures it does not exceed predefined limits.
• Adjusts for column-major layouts by calling SDDS_ReadBinaryColumns if necessary.
• Handles sparse reading by skipping rows based on sparse_interval and sparse_offset.
• If sparse_statistics is enabled, computes the specified statistics (average, median, min, max) on floating-point data.
• Handles errors by setting appropriate error messages and managing recovery modes.

Note

• This function provides extensive control over the reading process, allowing for optimized data access.
• Ensure that all parameters are set correctly to avoid unintended data skips or miscomputations.
• The function assumes that the dataset has been properly initialized and that the file pointers are correctly set up.
• Compression support (zLib for gzip, LZMA libraries) must be enabled during compilation for handling compressed files.

Definition at line 2202 of file SDDS_binary.c.

                                                                                                                                                              {
  int32_t n_rows32;
  int64_t n_rows, i, j, k, alloc_rows, rows_to_store, mod;
 
  /*  int32_t page_number, i; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
  void **statData=NULL;
  double statResult;
 
  if (SDDS_dataset->autoRecovered)
    return -1;
  if (SDDS_dataset->swapByteOrder) {
    return SDDS_ReadNonNativePageDetailed(SDDS_dataset, 0, sparse_interval, sparse_offset, last_rows);
  }
 
  /*  static char s[SDDS_MAXLINE]; */
  n_rows = 0;
  SDDS_SetReadRecoveryMode(SDDS_dataset, 0);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = SDDS_dataset->layout.gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = SDDS_dataset->layout.lzmafp;
    } else {
      fp = SDDS_dataset->layout.fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  if (!fBuffer->buffer) {
    if (defaultIOBufferSize == 0 && (SDDS_dataset->layout.popenUsed || !SDDS_dataset->layout.filename) && (sparse_interval > 1 || sparse_offset > 0 || last_rows > 0)) {
      SDDS_SetError("The IO buffer size is 0 for data being read from a pipe with sparsing.  This is not supported.");
      return 0;
    }
    if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * (defaultIOBufferSize + 1)))) {
      SDDS_SetError("Unable to do buffered read--allocation failure");
      return 0;
    }
    fBuffer->bufferSize = defaultIOBufferSize;
    fBuffer->bytesLeft = 0;
  }
  SDDS_dataset->rowcount_offset = -1;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!SDDS_GZipBufferedRead(&n_rows32, sizeof(n_rows32), gzfp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
      if (gzeof(gzfp))
        return (SDDS_dataset->page_number = -1);
      SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
      return (0);
    }
    if (n_rows32 == INT32_MIN) {
      if (!SDDS_GZipBufferedRead(&n_rows, sizeof(n_rows), gzfp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
        if (gzeof(gzfp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
        return (0);
      }
    } else {
      n_rows = n_rows32;
    }
  } else {
#endif
    /* This value will only be valid if read buffering is turned off, which is done for
     * certain append operations!   Should really modify SDDS_BufferedRead and SDDS_BufferedWrite
     * to provide ftell capability.
     */
    if (SDDS_dataset->layout.lzmaFile) {
      if (!SDDS_LZMABufferedRead(&n_rows32, sizeof(n_rows32), lzmafp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        if (lzma_eof(lzmafp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
        return (0);
      }
      if (n_rows32 == INT32_MIN) {
        if (!SDDS_LZMABufferedRead(&n_rows, sizeof(n_rows), lzmafp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
          if (lzma_eof(lzmafp))
            return (SDDS_dataset->page_number = -1);
          SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
          return (0);
        }
      } else {
        n_rows = n_rows32;
      }
    } else {
      SDDS_dataset->rowcount_offset = ftell(fp);
      if (!SDDS_BufferedRead(&n_rows32, sizeof(n_rows32), fp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        if (feof(fp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
        return (0);
      }
      if (n_rows32 == INT32_MIN) {
        if (!SDDS_BufferedRead(&n_rows, sizeof(n_rows), fp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
          if (feof(fp))
            return (SDDS_dataset->page_number = -1);
          SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadBinaryPageDetailed)");
          return (0);
        }
      } else {
        n_rows = n_rows32;
      }
    }
#if defined(zLib)
  }
#endif
 
#if defined(DEBUG)
  fprintf(stderr, "Expect %" PRId64 " rows of data\n", n_rows);
#endif
  if (n_rows < 0) {
    SDDS_SetError("Unable to read page--negative number of rows (SDDS_ReadBinaryPageDetailed)");
    return (0);
  }
  if (SDDS_dataset->layout.byteOrderDeclared == 0) {
    if (n_rows > 10000000) {
      SDDS_SetError("Unable to read page--endian byte order not declared and suspected to be non-native. (SDDS_ReadBinaryPageDetailed)");
      return (0);
    }
  }
  if (n_rows > SDDS_GetRowLimit()) {
    /* the number of rows is "unreasonably" large---treat like end-of-file */
    return (SDDS_dataset->page_number = -1);
  }
  if (last_rows < 0)
    last_rows = 0;
  /* Fix this limitation later */
  if (SDDS_dataset->layout.data_mode.column_major && sparse_statistics != 0) {
    SDDS_SetError("sparse_statistics is not yet supported for column-major layout. Use sddsconvert -majorOrder=row to convert first.\n");
    return (0);
  }
 
  if (last_rows) {
    sparse_interval = 1;
    sparse_offset = n_rows - last_rows;
  }
  if (sparse_interval <= 0)
    sparse_interval = 1;
  if (sparse_offset < 0)
    sparse_offset = 0;
 
  rows_to_store = (n_rows - sparse_offset) / sparse_interval + 2;
  alloc_rows = rows_to_store - SDDS_dataset->n_rows_allocated;
 
  if (!SDDS_StartPage(SDDS_dataset, 0) || !SDDS_LengthenTable(SDDS_dataset, alloc_rows)) {
    SDDS_SetError("Unable to read page--couldn't start page (SDDS_ReadBinaryPageDetailed)");
    return (0);
  }
 
  /* read the parameter values */
  if (!SDDS_ReadBinaryParameters(SDDS_dataset)) {
    SDDS_SetError("Unable to read page--parameter reading error (SDDS_ReadBinaryPageDetailed)");
    return (0);
  }
 
  /* read the array values */
  if (!SDDS_ReadBinaryArrays(SDDS_dataset)) {
    SDDS_SetError("Unable to read page--array reading error (SDDS_ReadBinaryPageDetailed)");
    return (0);
  }
  if (SDDS_dataset->layout.data_mode.column_major) {
    SDDS_dataset->n_rows = n_rows;
    if (!SDDS_ReadBinaryColumns(SDDS_dataset, sparse_interval, sparse_offset)) {
      SDDS_SetError("Unable to read page--column reading error (SDDS_ReadBinaryPageDetailed)");
      return (0);
    }
    return (SDDS_dataset->page_number);
  }
  if ((sparse_interval <= 1) && (sparse_offset == 0)) {
    for (j = 0; j < n_rows; j++) {
      if (!SDDS_ReadBinaryRow(SDDS_dataset, j, 0)) {
        SDDS_dataset->n_rows = j;
        if (SDDS_dataset->autoRecover) {
#if defined(DEBUG)
          fprintf(stderr, "Doing auto-read recovery\n");
#endif
          SDDS_dataset->autoRecovered = 1;
          SDDS_ClearErrors();
          return (SDDS_dataset->page_number);
        }
        SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadBinaryPageDetailed)");
        SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
        return (0);
      }
    }
    SDDS_dataset->n_rows = j;
    return (SDDS_dataset->page_number);
  } else {
    for (j = 0; j < sparse_offset; j++) {
      if (!SDDS_ReadBinaryRow(SDDS_dataset, 0, 1)) {
        SDDS_dataset->n_rows = 0;
        if (SDDS_dataset->autoRecover) {
          SDDS_dataset->autoRecovered = 1;
          SDDS_ClearErrors();
          return (SDDS_dataset->page_number);
        }
        SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadBinaryPageDetailed)");
        SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
        return (0);
      }
    }
    n_rows -= sparse_offset;
    if (sparse_statistics != 0) {
      // Allocate buffer space for statistical sparsing
      statData = (void**)malloc(SDDS_dataset->layout.n_columns * sizeof(void*));
      for (i = 0; i < SDDS_dataset->layout.n_columns; i++) {
        if (SDDS_FLOATING_TYPE(SDDS_dataset->layout.column_definition[i].type)) {
          // Not ideal for SDDS_LONGDOUBLE but we may never run across this error
          statData[i] = (double*)calloc(sparse_interval, sizeof(double));
        }
      }
      for (j = k = 0; j < n_rows; j++) {
        if (!SDDS_ReadBinaryRow(SDDS_dataset, k, 0)) {
          SDDS_dataset->n_rows = k;
          if (SDDS_dataset->autoRecover) {
            SDDS_dataset->autoRecovered = 1;
            SDDS_ClearErrors();
            return (SDDS_dataset->page_number);
          }
          SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadBinaryPageDetailed)");
          SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
          return (0);
        }
        for (i = 0; i < SDDS_dataset->layout.n_columns; i++) {
          switch (SDDS_dataset->layout.column_definition[i].type) {
          case SDDS_FLOAT:
            ((double*)statData[i])[j % sparse_interval] = (double)(((float*)SDDS_dataset->data[i])[k]);
            break;
          case SDDS_DOUBLE:
            ((double*)statData[i])[j % sparse_interval] = ((double*)SDDS_dataset->data[i])[k];
            break;
          case SDDS_LONGDOUBLE:
            ((double*)statData[i])[j % sparse_interval] = (double)(((long double*)SDDS_dataset->data[i])[k]);
            break;
          }
          if (SDDS_FLOATING_TYPE(SDDS_dataset->layout.column_definition[i].type)) {
            if (sparse_statistics == 1) {
              // Sparse and get average statistics
              compute_average(&statResult, (double*)statData[i], (j % sparse_interval) + 1);
            } else if (sparse_statistics == 2) {
              // Sparse and get median statistics
              compute_median(&statResult, (double*)statData[i], (j % sparse_interval) + 1);
            } else if (sparse_statistics == 3) {
              // Sparse and get minimum statistics
              statResult = min_in_array((double*)statData[i], (j % sparse_interval) + 1);
            } else if (sparse_statistics == 4) {
              // Sparse and get maximum statistics
              statResult = max_in_array((double*)statData[i], (j % sparse_interval) + 1);
            }
          }
          switch (SDDS_dataset->layout.column_definition[i].type) {
          case SDDS_FLOAT:
            ((float*)SDDS_dataset->data[i])[k] = statResult;
            break;
          case SDDS_DOUBLE:
            ((double*)SDDS_dataset->data[i])[k] = statResult;
            break;
          case SDDS_LONGDOUBLE:
            ((long double*)SDDS_dataset->data[i])[k] = statResult;
            break;
          }
        }
        if (j % sparse_interval == sparse_interval - 1) {
          k++;
        }
      }
      for (i = 0; i < SDDS_dataset->layout.n_columns; i++) {
        if (SDDS_FLOATING_TYPE(SDDS_dataset->layout.column_definition[i].type)) {
          free(statData[i]);
        }
      }
      free(statData);
    } else {
      for (j = k = 0; j < n_rows; j++) {
        if (!SDDS_ReadBinaryRow(SDDS_dataset, k, mod = j % sparse_interval)) {
          SDDS_dataset->n_rows = k;
          if (SDDS_dataset->autoRecover) {
            SDDS_dataset->autoRecovered = 1;
            SDDS_ClearErrors();
            return (SDDS_dataset->page_number);
          }
          SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadBinaryPageDetailed)");
          SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
          return (0);
        }
        k += mod ? 0 : 1;
      }
    }
    SDDS_dataset->n_rows = k;
    return (SDDS_dataset->page_number);
  }
}

SDDS_ReadBinaryPageLastRows()

int32_t SDDS_ReadBinaryPageLastRows	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	last_rows )

Reads the last specified number of rows from a binary page of an SDDS dataset.

This function reads the last last_rows rows from the binary page of the specified SDDS dataset. It is useful for retrieving recent data entries without processing the entire dataset.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
last_rows	The number of rows to read from the end of the dataset.

Returns

• Returns the page number on successful read.
• Returns -1 if the end-of-file is reached.
• Returns 0 on error.

The function internally calls SDDS_ReadBinaryPageDetailed with sparse_interval set to 1, sparse_offset set to 0, and last_rows as specified. This configuration ensures that only the last last_rows rows are read from the dataset.

Note

• This function is particularly useful for applications that need to display or process the most recent data entries.
• Ensure that last_rows does not exceed the total number of rows in the dataset to avoid errors.
• The function assumes that the dataset has been properly initialized and that the file pointers are correctly set up.

Definition at line 2150 of file SDDS_binary.c.

                                                                                   {
  return SDDS_ReadBinaryPageDetailed(SDDS_dataset, 1, 0, last_rows, 0);
}

SDDS_ReadBinaryParameters()

int32_t SDDS_ReadBinaryParameters

(

SDDS_DATASET *

SDDS_dataset

)

Reads binary parameters from the specified SDDS dataset.

This function iterates through all the parameters defined in the SDDS dataset layout and reads their values from the underlying file. It handles different data types, including strings, and manages memory allocation for string parameters. Depending on the dataset's compression settings (uncompressed, LZMA, or GZIP), it uses the appropriate reading functions to retrieve the parameter values.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successfully reading all binary parameters, or 0 if an error occurred.

Return values

1	All parameters were successfully read and stored.
0	An error occurred during the read operation, such as I/O errors, data type mismatches, or memory allocation failures.

Note

Parameters with the 'fixed_value' attribute are handled by scanning the fixed value string instead of reading from the file. String parameters are dynamically allocated and should be freed by the caller when no longer needed.

Definition at line 2947 of file SDDS_binary.c.

                                                              {
  int32_t i;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  char buffer[SDDS_MAXLINE];
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadBinaryParameters"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_parameters)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  for (i = 0; i < layout->n_parameters; i++) {
    if (layout->parameter_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
      continue;
    if (layout->parameter_definition[i].fixed_value) {
      strcpy(buffer, layout->parameter_definition[i].fixed_value);
      if (!SDDS_ScanData(buffer, layout->parameter_definition[i].type, 0, SDDS_dataset->parameter[i], 0, 1)) {
        SDDS_SetError("Unable to read page--parameter scanning error (SDDS_ReadBinaryParameters)");
        return (0);
      }
    } else if (layout->parameter_definition[i].type == SDDS_STRING) {
      if (*(char **)SDDS_dataset->parameter[i])
        free(*(char **)SDDS_dataset->parameter[i]);
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadGZipBinaryString(gzfp, fBuffer, 0))) {
          SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadBinaryParameters)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadLZMABinaryString(lzmafp, fBuffer, 0))) {
            SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadBinaryParameters)");
            return (0);
          }
        } else {
          if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadBinaryString(fp, fBuffer, 0))) {
            SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadBinaryParameters)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], gzfp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadBinaryParameters)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], lzmafp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadBinaryParameters)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], fp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadBinaryParameters)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
  return (1);
}

SDDS_ReadBinaryRow()

int32_t SDDS_ReadBinaryRow	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	row,
		int32_t	skip )

Reads a binary row from the specified SDDS dataset.

This function reads a single row of data from the given SDDS dataset. Depending on the dataset's configuration, it handles uncompressed, LZMA-compressed, or GZIP-compressed files. For each column in the dataset, the function reads the appropriate data type. If a column is of type string, it reads the string using the corresponding string reading function. If the 'skip' parameter is set, the function skips reading the data without storing it.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	row	The row number to read. Must be within the allocated range of rows in the dataset.
[in]	skip	If non-zero, the function skips reading the data for each column without storing it.

Returns

int32_t Returns 1 on successful reading of the row, or 0 if an error occurred.

Return values

1	The row was successfully read and stored (or skipped).
0	An error occurred during reading, such as I/O errors or memory allocation failures.

Note

This function may modify the dataset's data structures by allocating memory for string columns. Ensure that the dataset is properly initialized and that memory is managed appropriately.

Definition at line 2711 of file SDDS_binary.c.

                                                                                  {
  int64_t i, type, size;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadBinaryRow"))
    return (0);
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
 
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_columns; i++) {
      if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
        continue;
      if ((type = layout->column_definition[i].type) == SDDS_STRING) {
        if (!skip) {
          if (((char ***)SDDS_dataset->data)[i][row])
            free((((char ***)SDDS_dataset->data)[i][row]));
          if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
            return (0);
          }
        } else {
          if (!SDDS_ReadGZipBinaryString(gzfp, fBuffer, 1)) {
            SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
            return 0;
          }
        }
      } else {
        size = SDDS_type_size[type - 1];
        if (!SDDS_GZipBufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadBinaryRow)");
          return (0);
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_columns; i++) {
        if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
          continue;
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!skip) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
              return (0);
            }
          } else {
            if (!SDDS_ReadLZMABinaryString(lzmafp, fBuffer, 1)) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
              return 0;
            }
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_LZMABufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadBinaryRow)");
            return (0);
          }
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_columns; i++) {
        if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
          continue;
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!skip) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
              return (0);
            }
          } else {
            if (!SDDS_ReadBinaryString(fp, fBuffer, 1)) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadBinaryRows)");
              return 0;
            }
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_BufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadBinaryRow)");
            return (0);
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_ReadBinaryString()

char * SDDS_ReadBinaryString	(	FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	skip )

Reads a binary string from a file with buffering.

This function reads a binary string from the specified file by first reading the length of the string and then reading the string content based on the length. If the 'skip' parameter is set, the string data is skipped over instead of being stored. The function allocates memory for the string, which should be freed by the caller when no longer needed.

Parameters

[in]	fp	The file pointer to read from. Must be an open file in binary read mode.
[in,out]	fBuffer	Pointer to the file buffer used for buffered reading operations.
[in]	skip	If non-zero, the string data is skipped without being stored.

Returns

char* Returns a pointer to the read null-terminated string on success, or NULL if an error occurred.

Return values

NULL	An error occurred during reading or memory allocation.
Non-NULL	Pointer to the read string.

Definition at line 2616 of file SDDS_binary.c.

                                                                              {
  int32_t length;
  char *string;
 
  if (!SDDS_BufferedRead(&length, sizeof(length), fp, fBuffer, SDDS_LONG, 0) || length < 0)
    return (0);
  if (!(string = SDDS_Malloc(sizeof(*string) * (length + 1))))
    return (NULL);
  if (length && !SDDS_BufferedRead(skip ? NULL : string, sizeof(*string) * length, fp, fBuffer, SDDS_STRING, 0))
    return (NULL);
  string[length] = 0;
  return (string);
}

SDDS_ReadLZMABinaryString()

char * SDDS_ReadLZMABinaryString	(	struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	skip )

Reads a binary string from an LZMA-compressed file with buffering.

This function reads a binary string from the specified LZMA-compressed file by first reading the length of the string and then reading the string content based on the length. If the 'skip' parameter is set, the string data is skipped over instead of being stored. The function allocates memory for the string, which should be freed by the caller when no longer needed.

Parameters

[in]	lzmafp	The LZMA file pointer to read from. Must be an open LZMA-compressed file in read mode.
[in,out]	fBuffer	Pointer to the file buffer used for buffered reading operations.
[in]	skip	If non-zero, the string data is skipped without being stored.

Returns

char* Returns a pointer to the read null-terminated string on success, or NULL if an error occurred.

Return values

NULL	An error occurred during reading or memory allocation.
Non-NULL	Pointer to the read string.

Definition at line 2646 of file SDDS_binary.c.

                                                                                                 {
  int32_t length;
  char *string;
 
  if (!SDDS_LZMABufferedRead(&length, sizeof(length), lzmafp, fBuffer, SDDS_LONG, 0) || length < 0)
    return (0);
  if (!(string = SDDS_Malloc(sizeof(*string) * (length + 1))))
    return (NULL);
  if (length && !SDDS_LZMABufferedRead(skip ? NULL : string, sizeof(*string) * length, lzmafp, fBuffer, SDDS_STRING, 0))
    return (NULL);
  string[length] = 0;
  return (string);
}

SDDS_ReadNewBinaryRows()

int32_t SDDS_ReadNewBinaryRows

(

SDDS_DATASET *

SDDS_dataset

)

Reads new binary rows from the SDDS dataset.

This function updates the SDDS dataset by reading any new rows that have been added to the underlying file since the last read operation. It verifies that the dataset is in a compatible binary format and ensures that byte order and compression settings are supported. If the number of rows in the file exceeds the currently allocated rows in memory, the function expands the dataset's internal storage to accommodate the new rows.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns the number of new rows successfully read on success, or -1 if an error occurred.

Return values

>0	The number of new rows read and added to the dataset.
-1	An error occurred during the read operation, such as unsupported file format, I/O errors, or memory allocation failures.

Note

This function does not support MPI parallel I/O, ASCII files, column-major order binary files, non-native byte orders, or compressed files (gzip or lzma). Attempts to use these features will result in an error.

Definition at line 2837 of file SDDS_binary.c.

                                                           {
  int64_t row, offset, newRows = 0;
  int32_t rowsPresent32;
  int64_t rowsPresent;
 
#if SDDS_MPI_IO
  if (SDDS_dataset->parallel_io) {
    SDDS_SetError("Error: MPI mode not supported yet in SDDS_ReadNewBinaryRows");
    return -1;
  }
#endif
  if (SDDS_dataset->original_layout.data_mode.mode == SDDS_ASCII) {
    SDDS_SetError("Error: ASCII files not supported in SDDS_ReadNewBinaryRows");
    return -1;
  }
  if (SDDS_dataset->layout.data_mode.column_major) {
    SDDS_SetError("Error: column-major order binary files not supported in SDDS_ReadNewBinaryRows");
    return -1;
  }
  if (SDDS_dataset->swapByteOrder) {
    SDDS_SetError("Error: Non-native endian not supported yet in SDDS_ReadNewBinaryRows");
    return -1;
  }
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    SDDS_SetError("Error: gzip compressed files not supported yet in SDDS_ReadNewBinaryRows");
    return -1;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      SDDS_SetError("Error: lzma compressed files not supported yet in SDDS_ReadNewBinaryRows");
      return -1;
    }
#if defined(zLib)
  }
#endif
 
  // Read how many rows we have now
  offset = ftell(SDDS_dataset->layout.fp);
  fseek(SDDS_dataset->layout.fp, SDDS_dataset->rowcount_offset, 0);
  if (SDDS_dataset->layout.data_mode.mode == SDDS_BINARY) {
    if (fread(&rowsPresent32, sizeof(rowsPresent32), 1, SDDS_dataset->layout.fp) == 0) {
      SDDS_SetError("Error: row count not present or not correct length");
      return -1;
    }
    if (SDDS_dataset->swapByteOrder) {
      SDDS_SwapLong(&rowsPresent32);
    }
    if (rowsPresent32 == INT32_MIN) {
      if (fread(&rowsPresent, sizeof(rowsPresent), 1, SDDS_dataset->layout.fp) == 0) {
        SDDS_SetError("Error: row count not present or not correct length");
        return -1;
      }
      if (SDDS_dataset->swapByteOrder) {
        SDDS_SwapLong64(&rowsPresent);
      }
    } else {
      rowsPresent = rowsPresent32;
    }
  } else {
    char buffer[30];
    if (!fgets(buffer, 30, SDDS_dataset->layout.fp) || strlen(buffer) != 21 || sscanf(buffer, "%" SCNd64, &rowsPresent) != 1) {
      SDDS_SetError("Error: row count not present or not correct length");
      return -1;
    }
  }
  fseek(SDDS_dataset->layout.fp, offset, 0);
 
  // If the row count listed in the file is greather than the allocated rows, then lengthen the table in memory
  if (rowsPresent > SDDS_dataset->n_rows_allocated) {
    if (!SDDS_LengthenTable(SDDS_dataset, rowsPresent + 3)) {
      return -1;
    }
  }
 
  for (row = SDDS_dataset->n_rows; row < rowsPresent; row++) {
    if (!SDDS_ReadBinaryRow(SDDS_dataset, row, 0)) {
      if (SDDS_dataset->autoRecover) {
        row--;
        SDDS_dataset->autoRecovered = 1;
        SDDS_ClearErrors();
        break;
      }
      SDDS_SetError("Unable to read page--error reading data row");
      return -1;
    }
  }
  newRows = row + 1 - SDDS_dataset->n_rows;
  SDDS_dataset->n_rows = row + 1;
  return newRows;
}

SDDS_ReadNonNativeBinaryArrays()

int32_t SDDS_ReadNonNativeBinaryArrays

(

SDDS_DATASET *

SDDS_dataset

)

Reads non-native endian binary arrays from an SDDS dataset.

This function iterates through all array definitions in the specified SDDS dataset and reads their binary data from the underlying file. It handles various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. For string arrays, it reads each string individually, ensuring proper memory allocation and byte order conversion. The function supports different compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. After reading, it swaps the endianness of the array data to match the system's native byte order.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successful reading and byte-swapping of all arrays, or 0 if an error occurred.

Return values

1	All non-native endian arrays were successfully read and byte-swapped.
0	An error occurred during the read or byte-swapping process, such as I/O failures, memory allocation issues, or corrupted array definitions.

Note

This function modifies the dataset's array data in place. It should be called after successfully opening and preparing the dataset for reading. Ensure that the dataset structure is properly initialized to prevent undefined behavior.

Definition at line 4582 of file SDDS_binary.c.

                                                                   {
  int32_t i, j;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  /*  static char buffer[SDDS_MAXLINE]; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_ARRAY *array;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativeBinaryArrays"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_arrays)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  if (!SDDS_dataset->array) {
    SDDS_SetError("Unable to read array--pointer to structure storage area is NULL (SDDS_ReadNonNativeBinaryArrays)");
    return (0);
  }
  for (i = 0; i < layout->n_arrays; i++) {
    array = SDDS_dataset->array + i;
    if (array->definition && !SDDS_FreeArrayDefinition(array->definition)) {
      SDDS_SetError("Unable to get array--array definition corrupted (SDDS_ReadNonNativeBinaryArrays)");
      return (0);
    }
    if (!SDDS_CopyArrayDefinition(&array->definition, layout->array_definition + i)) {
      SDDS_SetError("Unable to read array--definition copy failed (SDDS_ReadNonNativeBinaryArrays)");
      return (0);
    }
    /*if (array->dimension) free(array->dimension); */
    if (!(array->dimension = SDDS_Realloc(array->dimension, sizeof(*array->dimension) * array->definition->dimensions))) {
      SDDS_SetError("Unable to read array--allocation failure (SDDS_ReadNonNativeBinaryArrays)");
      return (0);
    }
#if defined(zLib)
    if (SDDS_dataset->layout.gzipFile) {
      if (!SDDS_GZipBufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, gzfp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadNonNativeBinaryArrays)");
        return (0);
      }
    } else {
#endif
      if (SDDS_dataset->layout.lzmaFile) {
        if (!SDDS_LZMABufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, lzmafp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadNonNativeBinaryArrays)");
          return (0);
        }
      } else {
        if (!SDDS_BufferedRead(array->dimension, sizeof(*array->dimension) * array->definition->dimensions, fp, fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading dimensions (SDDS_ReadNonNativeBinaryArrays)");
          return (0);
        }
      }
#if defined(zLib)
    }
#endif
    array->elements = 1;
    for (j = 0; j < array->definition->dimensions; j++) {
      SDDS_SwapLong(&(array->dimension[j]));
      array->elements *= array->dimension[j];
    }
    if (array->data)
      free(array->data);
    array->data = array->pointer = NULL;
    if (array->elements == 0)
      continue;
    if (array->elements < 0) {
      SDDS_SetError("Unable to read array--number of elements is negative (SDDS_ReadNonNativeBinaryArrays)");
      return (0);
    }
    if (!(array->data = SDDS_Realloc(array->data, array->elements * SDDS_type_size[array->definition->type - 1]))) {
      SDDS_SetError("Unable to read array--allocation failure (SDDS_ReadNonNativeBinaryArrays)");
      return (0);
    }
    if (array->definition->type == SDDS_STRING) {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        for (j = 0; j < array->elements; j++) {
          if (!(((char **)(array->data))[j] = SDDS_ReadNonNativeGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadNonNativeBinaryArrays)");
            return (0);
          }
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          for (j = 0; j < array->elements; j++) {
            if (!(((char **)(array->data))[j] = SDDS_ReadNonNativeLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadNonNativeBinaryArrays)");
              return (0);
            }
          }
        } else {
          for (j = 0; j < array->elements; j++) {
            if (!(((char **)(array->data))[j] = SDDS_ReadNonNativeBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read arrays--failure reading string (SDDS_ReadNonNativeBinaryArrays)");
              return (0);
            }
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, gzfp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadNonNativeBinaryArrays)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, lzmafp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadNonNativeBinaryArrays)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(array->data, SDDS_type_size[array->definition->type - 1] * array->elements, fp, fBuffer, array->definition->type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read arrays--failure reading values (SDDS_ReadNonNativeBinaryArrays)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
  SDDS_SwapEndsArrayData(SDDS_dataset);
  return (1);
}

SDDS_ReadNonNativeBinaryColumns()

int32_t SDDS_ReadNonNativeBinaryColumns

(

SDDS_DATASET *

SDDS_dataset

)

Reads the non-native endian binary columns from an SDDS dataset.

This function is similar to SDDS_ReadBinaryColumns but specifically handles columns with non-native endianness. It iterates through all column definitions within the specified SDDS dataset and reads their binary data from the underlying file, ensuring that the byte order is correctly swapped to match the system's native endianness. The function supports various compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successful reading and byte-swapping of all columns, or 0 if an error occurred.

Return values

1	All non-native endian columns were successfully read and byte-swapped.
0	An error occurred during the read or byte-swapping operation, such as I/O failures, memory allocation issues, or corrupted column definitions.

Note

This function assumes that the dataset's byte order has been declared and that the underlying file's byte order differs from the system's native byte order. Proper initialization and configuration of the SDDS_dataset structure are required before calling this function.

Definition at line 3465 of file SDDS_binary.c.

                                                                    {
  int64_t i, row;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  /*  static char buffer[SDDS_MAXLINE]; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativeBinaryColumns"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_columns || !SDDS_dataset->n_rows)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
 
  for (i = 0; i < layout->n_columns; i++) {
    if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
      continue;
    if (layout->column_definition[i].type == SDDS_STRING) {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        for (row = 0; row < SDDS_dataset->n_rows; row++) {
          if (((char ***)SDDS_dataset->data)[i][row])
            free((((char ***)SDDS_dataset->data)[i][row]));
          if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadNonNativeBinaryColumns)");
            return (0);
          }
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadNonNativeBinaryColumms)");
              return (0);
            }
          }
        } else {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read columns--failure reading string (SDDS_ReadNonNativeBinaryColumms)");
              return (0);
            }
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, gzfp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadNonNativeBinaryColumns)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, lzmafp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadNonNativeBinaryColumns)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(SDDS_dataset->data[i], SDDS_type_size[layout->column_definition[i].type - 1] * SDDS_dataset->n_rows, fp, fBuffer, layout->column_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read columns--failure reading values (SDDS_ReadNonNativeBinaryColumns)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
  return (1);
}

SDDS_ReadNonNativeBinaryPage()

int32_t SDDS_ReadNonNativeBinaryPage	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	sparse_interval,
		int64_t	sparse_offset )

Reads a non-native endian binary page from an SDDS dataset.

This function reads a binary page from the specified SDDS dataset, handling data with non-native endianness. It performs necessary byte order conversions to ensure correct data interpretation on the host system. The function supports sparse reading based on the provided interval and offset.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	sparse_interval	Interval between rows to be read for sparsity.
[in]	sparse_offset	Offset to start reading rows for sparsity.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function is a wrapper for SDDS_ReadNonNativeBinaryPageDetailed with specific parameters.

Definition at line 4187 of file SDDS_binary.c.

                                                                                                                 {
  return SDDS_ReadNonNativeBinaryPageDetailed(SDDS_dataset, sparse_interval, sparse_offset, 0);
}

SDDS_ReadNonNativeBinaryPageDetailed()

int32_t SDDS_ReadNonNativeBinaryPageDetailed	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	sparse_interval,
		int64_t	sparse_offset,
		int64_t	last_rows )

Reads a detailed non-native endian binary page from an SDDS dataset.

This function reads a binary page from the specified SDDS dataset, handling data with non-native endianness. It supports both sparse reading and reading of the last few rows based on the provided parameters. The function performs necessary byte order conversions to ensure correct data interpretation on the host system.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	sparse_interval	Interval between rows to be read for sparsity.
[in]	sparse_offset	Offset to start reading rows for sparsity.
[in]	last_rows	Number of last rows to read from the dataset.

Returns

int32_t Returns the page number on success, or 0 on failure.

Return values

>0	Page number successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function handles various compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. It manages memory allocation for parameters, arrays, and columns, ensuring that data is correctly stored and byte-swapped as necessary. The function also updates the dataset's row count and handles auto-recovery in case of errors.

Definition at line 4236 of file SDDS_binary.c.

                                                                                                                                            {
  int32_t n_rows32 = 0;
  int64_t n_rows, j, k, alloc_rows, rows_to_store, mod;
  /*  int32_t page_number, i; */
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
 
  /*  static char s[SDDS_MAXLINE]; */
  n_rows = 0;
  SDDS_SetReadRecoveryMode(SDDS_dataset, 0);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = SDDS_dataset->layout.gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = SDDS_dataset->layout.lzmafp;
    } else {
      fp = SDDS_dataset->layout.fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  if (!fBuffer->buffer) {
    if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * defaultIOBufferSize))) {
      SDDS_SetError("Unable to do buffered read--allocation failure");
      return 0;
    }
    fBuffer->bufferSize = defaultIOBufferSize;
    fBuffer->bytesLeft = 0;
  }
  SDDS_dataset->rowcount_offset = -1;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!SDDS_GZipBufferedRead(&n_rows32, sizeof(n_rows32), gzfp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
      if (gzeof(gzfp))
        return (SDDS_dataset->page_number = -1);
      SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
      return (0);
    }
    SDDS_SwapLong(&n_rows32);
    if (n_rows32 == INT32_MIN) {
      if (!SDDS_GZipBufferedRead(&n_rows, sizeof(n_rows), gzfp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
        if (gzeof(gzfp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
        return (0);
      }
      SDDS_SwapLong64(&n_rows);
    } else {
      n_rows = n_rows32;
    }
  } else {
#endif
    /* This value will only be valid if read buffering is turned off, which is done for
     * certain append operations!   Should really modify SDDS_BufferedRead and SDDS_BufferedWrite
     * to provide ftell capability.
     */
    if (SDDS_dataset->layout.lzmaFile) {
      if (!SDDS_LZMABufferedRead(&n_rows32, sizeof(n_rows32), lzmafp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        if (lzma_eof(lzmafp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
        return (0);
      }
      SDDS_SwapLong(&n_rows32);
      if (n_rows32 == INT32_MIN) {
        if (!SDDS_LZMABufferedRead(&n_rows, sizeof(n_rows), lzmafp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
          if (lzma_eof(lzmafp))
            return (SDDS_dataset->page_number = -1);
          SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&n_rows);
      } else {
        n_rows = n_rows32;
      }
    } else {
      SDDS_dataset->rowcount_offset = ftell(fp);
      if (!SDDS_BufferedRead(&n_rows32, sizeof(n_rows32), fp, &SDDS_dataset->fBuffer, SDDS_LONG, SDDS_dataset->layout.byteOrderDeclared)) {
        if (feof(fp))
          return (SDDS_dataset->page_number = -1);
        SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
        return (0);
      }
      SDDS_SwapLong(&n_rows32);
      if (n_rows32 == INT32_MIN) {
        if (!SDDS_BufferedRead(&n_rows, sizeof(n_rows), fp, &SDDS_dataset->fBuffer, SDDS_LONG64, SDDS_dataset->layout.byteOrderDeclared)) {
          if (feof(fp))
            return (SDDS_dataset->page_number = -1);
          SDDS_SetError("Unable to read page--failure reading number of rows (SDDS_ReadNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&n_rows);
      } else {
        n_rows = n_rows32;
      }
    }
#if defined(zLib)
  }
#endif
  if (n_rows < 0) {
    SDDS_SetError("Unable to read page--negative number of rows (SDDS_ReadNonNativeBinaryPage)");
    return (0);
  }
  if (last_rows < 0)
    last_rows = 0;
  /* Fix this limitation later */
  if (SDDS_dataset->layout.data_mode.column_major) {
    sparse_interval = 1;
    sparse_offset = 0;
    last_rows = 0;
  }
  if (last_rows) {
    sparse_interval = 1;
    sparse_offset = n_rows - last_rows;
    rows_to_store = last_rows + 2;
    alloc_rows = rows_to_store - SDDS_dataset->n_rows_allocated;
  }
  if (sparse_interval <= 0)
    sparse_interval = 1;
  if (sparse_offset < 0)
    sparse_offset = 0;
 
  rows_to_store = (n_rows - sparse_offset) / sparse_interval + 2;
  alloc_rows = rows_to_store - SDDS_dataset->n_rows_allocated;
  if (!SDDS_StartPage(SDDS_dataset, 0) || !SDDS_LengthenTable(SDDS_dataset, alloc_rows)) {
    SDDS_SetError("Unable to read page--couldn't start page (SDDS_ReadNonNativeBinaryPage)");
    return (0);
  }
 
  /* read the parameter values */
  if (!SDDS_ReadNonNativeBinaryParameters(SDDS_dataset)) {
    SDDS_SetError("Unable to read page--parameter reading error (SDDS_ReadNonNativeBinaryPage)");
    return (0);
  }
 
  /* read the array values */
  if (!SDDS_ReadNonNativeBinaryArrays(SDDS_dataset)) {
    SDDS_SetError("Unable to read page--array reading error (SDDS_ReadNonNativeBinaryPage)");
    return (0);
  }
  if (SDDS_dataset->layout.data_mode.column_major) {
    SDDS_dataset->n_rows = n_rows;
    if (!SDDS_ReadNonNativeBinaryColumns(SDDS_dataset)) {
      SDDS_SetError("Unable to read page--column reading error (SDDS_ReadNonNativeBinaryPage)");
      return (0);
    }
    SDDS_SwapEndsColumnData(SDDS_dataset);
    return (SDDS_dataset->page_number);
  }
  if ((sparse_interval <= 1) && (sparse_offset == 0)) {
    for (j = 0; j < n_rows; j++) {
      if (!SDDS_ReadNonNativeBinaryRow(SDDS_dataset, j, 0)) {
        SDDS_dataset->n_rows = j - 1;
        if (SDDS_dataset->autoRecover) {
          SDDS_ClearErrors();
          SDDS_SwapEndsColumnData(SDDS_dataset);
          return (SDDS_dataset->page_number);
        }
        SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadNonNativeBinaryPage)");
        SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
        return (0);
      }
    }
    SDDS_dataset->n_rows = j;
    SDDS_SwapEndsColumnData(SDDS_dataset);
    return (SDDS_dataset->page_number);
  } else {
    for (j = 0; j < sparse_offset; j++) {
      if (!SDDS_ReadNonNativeBinaryRow(SDDS_dataset, 0, 1)) {
        SDDS_dataset->n_rows = 0;
        if (SDDS_dataset->autoRecover) {
          SDDS_ClearErrors();
          SDDS_SwapEndsColumnData(SDDS_dataset);
          return (SDDS_dataset->page_number);
        }
        SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadNonNativeBinaryPage)");
        SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
        return (0);
      }
    }
    n_rows -= sparse_offset;
    for (j = k = 0; j < n_rows; j++) {
      if (!SDDS_ReadNonNativeBinaryRow(SDDS_dataset, k, mod = j % sparse_interval)) {
        SDDS_dataset->n_rows = k - 1;
        if (SDDS_dataset->autoRecover) {
          SDDS_ClearErrors();
          SDDS_SwapEndsColumnData(SDDS_dataset);
          return (SDDS_dataset->page_number);
        }
        SDDS_SetError("Unable to read page--error reading data row (SDDS_ReadNonNativeBinaryPage)");
        SDDS_SetReadRecoveryMode(SDDS_dataset, 1);
        return (0);
      }
      k += mod ? 0 : 1;
    }
    SDDS_dataset->n_rows = k;
    SDDS_SwapEndsColumnData(SDDS_dataset);
    return (SDDS_dataset->page_number);
  }
}

SDDS_ReadNonNativeBinaryPageLastRows()

int32_t SDDS_ReadNonNativeBinaryPageLastRows	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	last_rows )

Reads the last few rows from a non-native endian binary page in an SDDS dataset.

This function reads the specified number of last rows from a binary page in the given SDDS dataset, handling data with non-native endianness. It performs necessary byte order conversions to ensure correct data interpretation on the host system.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	last_rows	Number of last rows to read from the dataset.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function is a wrapper for SDDS_ReadNonNativeBinaryPageDetailed with specific parameters to read the last few rows. It should be used when only the most recent rows are needed.

Definition at line 4209 of file SDDS_binary.c.

                                                                                            {
  return SDDS_ReadNonNativeBinaryPageDetailed(SDDS_dataset, 1, 0, last_rows);
}

SDDS_ReadNonNativeBinaryParameters()

int32_t SDDS_ReadNonNativeBinaryParameters

(

SDDS_DATASET *

SDDS_dataset

)

Reads non-native endian binary parameters from an SDDS dataset.

This function iterates through all parameter definitions in the specified SDDS dataset and reads their binary data from the underlying file. It handles various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. For string parameters, it reads each string individually, ensuring proper memory allocation and byte order conversion. Parameters with fixed values are processed by scanning the fixed value strings into the appropriate data types. The function supports different compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns 1 on successful reading and byte-swapping of all parameters, or 0 if an error occurred.

Return values

1	All non-native endian parameters were successfully read and byte-swapped.
0	An error occurred during the read or byte-swapping process, such as I/O failures, memory allocation issues, or corrupted parameter definitions.

Note

This function modifies the dataset's parameter data in place. It should be called after successfully opening and preparing the dataset for reading. Ensure that the dataset structure is properly initialized to prevent undefined behavior.

Definition at line 4466 of file SDDS_binary.c.

                                                                       {
  int32_t i;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  char buffer[SDDS_MAXLINE];
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
  FILE *fp = NULL;
  struct lzmafile *lzmafp = NULL;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativeBinaryParameters"))
    return (0);
  layout = &SDDS_dataset->layout;
  if (!layout->n_parameters)
    return (1);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
    } else {
      fp = layout->fp;
    }
#if defined(zLib)
  }
#endif
  fBuffer = &SDDS_dataset->fBuffer;
  for (i = 0; i < layout->n_parameters; i++) {
    if (layout->parameter_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
      continue;
    if (layout->parameter_definition[i].fixed_value) {
      strcpy(buffer, layout->parameter_definition[i].fixed_value);
      if (!SDDS_ScanData(buffer, layout->parameter_definition[i].type, 0, SDDS_dataset->parameter[i], 0, 1)) {
        SDDS_SetError("Unable to read page--parameter scanning error (SDDS_ReadNonNativeBinaryParameters)");
        return (0);
      }
    } else if (layout->parameter_definition[i].type == SDDS_STRING) {
      if (*(char **)SDDS_dataset->parameter[i])
        free(*(char **)SDDS_dataset->parameter[i]);
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadNonNativeGZipBinaryString(gzfp, fBuffer, 0))) {
          SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadNonNativeBinaryParameters)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadNonNativeLZMABinaryString(lzmafp, fBuffer, 0))) {
            SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadNonNativeBinaryParameters)");
            return (0);
          }
        } else {
          if (!(*((char **)SDDS_dataset->parameter[i]) = SDDS_ReadNonNativeBinaryString(fp, fBuffer, 0))) {
            SDDS_SetError("Unable to read parameters--failure reading string (SDDS_ReadNonNativeBinaryParameters)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    } else {
#if defined(zLib)
      if (SDDS_dataset->layout.gzipFile) {
        if (!SDDS_GZipBufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], gzfp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadNonNativeBinaryParameters)");
          return (0);
        }
      } else {
#endif
        if (SDDS_dataset->layout.lzmaFile) {
          if (!SDDS_LZMABufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], lzmafp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadNonNativeBinaryParameters)");
            return (0);
          }
        } else {
          if (!SDDS_BufferedRead(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], fp, fBuffer, layout->parameter_definition[i].type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read parameters--failure reading value (SDDS_ReadNonNativeBinaryParameters)");
            return (0);
          }
        }
#if defined(zLib)
      }
#endif
    }
  }
  SDDS_SwapEndsParameterData(SDDS_dataset);
  return (1);
}

SDDS_ReadNonNativeBinaryRow()

int32_t SDDS_ReadNonNativeBinaryRow	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	row,
		int32_t	skip )

Reads a non-native endian binary row from an SDDS dataset.

This function reads a single row of data from the specified SDDS dataset, handling data with non-native endianness. It iterates through all column definitions and reads each column's data for the given row. For string columns, it ensures proper memory allocation and byte order conversion. For other data types, it reads the binary data and performs necessary byte swapping. The function supports different compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	row	The index of the row to read.
[in]	skip	If non-zero, the function will skip reading the row data, useful for sparse reading.

Returns

int32_t Returns 1 on successful reading of the row, or 0 if an error occurred.

Return values

1	The row was successfully read and byte-swapped.
0	An error occurred during the read or byte-swapping process, such as I/O failures or corrupted data.

Note

This function modifies the dataset's data in place. It should be called after successfully opening and preparing the dataset for reading. Ensure that the dataset structure is properly initialized to prevent undefined behavior.

Definition at line 4752 of file SDDS_binary.c.

                                                                                           {
  int64_t i, type, size;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativeBinaryRow"))
    return (0);
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
 
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_columns; i++) {
      if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
        continue;
      if ((type = layout->column_definition[i].type) == SDDS_STRING) {
        if (!skip) {
          if (((char ***)SDDS_dataset->data)[i][row])
            free((((char ***)SDDS_dataset->data)[i][row]));
          if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeGZipBinaryString(gzfp, fBuffer, 0))) {
            SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
            return (0);
          }
        } else {
          if (!SDDS_ReadNonNativeGZipBinaryString(gzfp, fBuffer, 1)) {
            SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
            return 0;
          }
        }
      } else {
        size = SDDS_type_size[type - 1];
        if (!SDDS_GZipBufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
          SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadNonNativeBinaryRow)");
          return (0);
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_columns; i++) {
        if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
          continue;
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!skip) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeLZMABinaryString(lzmafp, fBuffer, 0))) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
              return (0);
            }
          } else {
            if (!SDDS_ReadNonNativeLZMABinaryString(lzmafp, fBuffer, 1)) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
              return 0;
            }
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_LZMABufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadNonNativeBinaryRow)");
            return (0);
          }
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_columns; i++) {
        if (layout->column_definition[i].definition_mode & SDDS_WRITEONLY_DEFINITION)
          continue;
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!skip) {
            if (((char ***)SDDS_dataset->data)[i][row])
              free((((char ***)SDDS_dataset->data)[i][row]));
            if (!(((char ***)SDDS_dataset->data)[i][row] = SDDS_ReadNonNativeBinaryString(fp, fBuffer, 0))) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
              return (0);
            }
          } else {
            if (!SDDS_ReadNonNativeBinaryString(fp, fBuffer, 1)) {
              SDDS_SetError("Unable to read rows--failure reading string (SDDS_ReadNonNativeBinaryRow)");
              return 0;
            }
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_BufferedRead(skip ? NULL : (char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer, type, SDDS_dataset->layout.byteOrderDeclared)) {
            SDDS_SetError("Unable to read row--failure reading value (SDDS_ReadNonNativeBinaryRow)");
            return (0);
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_ReadNonNativeBinaryString()

char * SDDS_ReadNonNativeBinaryString	(	FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	skip )

Reads a non-native endian binary string from a file.

This function reads a binary string from the specified file pointer, handling non-native endianness. It first reads the length of the string, swaps its byte order if necessary, allocates memory for the string, reads the string data, and null-terminates it.

Parameters

[in]	fp	Pointer to the FILE from which to read the string.
[in,out]	fBuffer	Pointer to the SDDS_FILEBUFFER structure used for buffered reading.
[in]	skip	If non-zero, the function will skip reading the string data, useful for sparse reading.

Returns

char* Returns a pointer to the read string on success, or NULL if an error occurred.

Return values

Non-NULL	Pointer to the newly allocated string.
NULL	An error occurred during reading or memory allocation.

Note

The caller is responsible for freeing the returned string to prevent memory leaks.

Definition at line 4875 of file SDDS_binary.c.

                                                                                       {
  int32_t length;
  char *string;
 
  if (!SDDS_BufferedRead(&length, sizeof(length), fp, fBuffer, SDDS_LONG, 0))
    return (0);
  SDDS_SwapLong(&length);
  if (length < 0)
    return (0);
  if (!(string = SDDS_Malloc(sizeof(*string) * (length + 1))))
    return (NULL);
  if (length && !SDDS_BufferedRead(skip ? NULL : string, sizeof(*string) * length, fp, fBuffer, SDDS_STRING, 0))
    return (NULL);
  string[length] = 0;
  return (string);
}

SDDS_ReadNonNativeLZMABinaryString()

char * SDDS_ReadNonNativeLZMABinaryString	(	struct lzmafile *	lzmafp,
		SDDS_FILEBUFFER *	fBuffer,
		int32_t	skip )

Reads a non-native endian binary string from an LZMA-compressed file.

This function reads a binary string from the specified LZMA-compressed file pointer, handling non-native endianness. It first reads the length of the string, swaps its byte order if necessary, allocates memory for the string, reads the string data, and null-terminates it.

Parameters

[in]	lzmafp	Pointer to the LZMAFILE from which to read the string.
[in,out]	fBuffer	Pointer to the SDDS_FILEBUFFER structure used for buffered reading.
[in]	skip	If non-zero, the function will skip reading the string data, useful for sparse reading.

Returns

char* Returns a pointer to the read string on success, or NULL if an error occurred.

Return values

Non-NULL	Pointer to the newly allocated string.
NULL	An error occurred during reading or memory allocation.

Note

The caller is responsible for freeing the returned string to prevent memory leaks.

Definition at line 4909 of file SDDS_binary.c.

                                                                                                          {
  int32_t length;
  char *string;
 
  if (!SDDS_LZMABufferedRead(&length, sizeof(length), lzmafp, fBuffer, SDDS_LONG, 0))
    return (0);
  SDDS_SwapLong(&length);
  if (length < 0)
    return (0);
  if (!(string = SDDS_Malloc(sizeof(*string) * (length + 1))))
    return (NULL);
  if (length && !SDDS_LZMABufferedRead(skip ? NULL : string, sizeof(*string) * length, lzmafp, fBuffer, SDDS_STRING, 0))
    return (NULL);
  string[length] = 0;
  return (string);
}

SDDS_ReadNonNativePage()

int32_t SDDS_ReadNonNativePage

(

SDDS_DATASET *

SDDS_dataset

)

Reads a non-native endian page from an SDDS dataset.

This function reads a page of data from the specified SDDS dataset, handling data with non-native endianness. It supports both ASCII and binary data modes, performing necessary byte order conversions to ensure correct data interpretation on the host system.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to read from.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function is a wrapper for SDDS_ReadNonNativePageDetailed with default parameters. It should be used when no specific mode, sparse interval, or offset is required.

Definition at line 4008 of file SDDS_binary.c.

                                                           {
  return SDDS_ReadNonNativePageDetailed(SDDS_dataset, 0, 1, 0, 0);
}

SDDS_ReadNonNativePageDetailed()

int32_t SDDS_ReadNonNativePageDetailed	(	SDDS_DATASET *	SDDS_dataset,
		uint32_t	mode,
		int64_t	sparse_interval,
		int64_t	sparse_offset,
		int64_t	last_rows )

Reads a detailed non-native endian page from an SDDS dataset.

This function reads a page of data from the specified SDDS dataset, handling data with non-native endianness. It supports both ASCII and binary data modes, performing necessary byte order conversions to ensure correct data interpretation on the host system. Additionally, it allows for sparse reading and reading of the last few rows based on the provided parameters.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	mode	Mode flag to support future expansion.
[in]	sparse_interval	Interval between rows to be read for sparsity.
[in]	sparse_offset	Offset to start reading rows for sparsity.
[in]	last_rows	Number of last rows to read from the dataset.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function handles various compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. It manages memory allocation for parameters, arrays, and columns, ensuring that data is correctly stored and byte-swapped as necessary.

Definition at line 4057 of file SDDS_binary.c.

{
  int32_t retval;
  /*  SDDS_LAYOUT layout_copy; */
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativePageDetailed"))
    return (0);
  if (SDDS_dataset->layout.disconnected) {
    SDDS_SetError("Can't read page--file is disconnected (SDDS_ReadNonNativePageDetailed)");
    return 0;
  }
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!SDDS_dataset->layout.gzfp) {
      SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageDetailed)");
      return (0);
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!SDDS_dataset->layout.lzmafp) {
        SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageDetailed)");
        return (0);
      }
    } else {
      if (!SDDS_dataset->layout.fp) {
        SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageDetailed)");
        return (0);
      }
    }
#if defined(zLib)
  }
#endif
  if (SDDS_dataset->original_layout.data_mode.mode == SDDS_ASCII) {
    if ((retval = SDDS_ReadAsciiPage(SDDS_dataset, sparse_interval, sparse_offset, 0)) < 1) {
      return (retval);
    }
  } else if (SDDS_dataset->original_layout.data_mode.mode == SDDS_BINARY) {
    if ((retval = SDDS_ReadNonNativeBinaryPage(SDDS_dataset, sparse_interval, sparse_offset)) < 1) {
      return (retval);
    }
  } else {
    SDDS_SetError("Unable to read page--unrecognized data mode (SDDS_ReadNonNativePageDetailed)");
    return (0);
  }
  return (retval);
}

SDDS_ReadNonNativePageLastRows()

int32_t SDDS_ReadNonNativePageLastRows	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	last_rows )

Reads the last few rows from a non-native endian page in an SDDS dataset.

This function reads the specified number of last rows from the non-native endian page of the given SDDS dataset. It handles data with non-native endianness, performing necessary byte order conversions to ensure correct data interpretation on the host system.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	last_rows	Number of last rows to read from the dataset.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function is a wrapper for SDDS_ReadNonNativePageDetailed with specific parameters to read the last few rows. It should be used when only the most recent rows are needed.

Definition at line 4124 of file SDDS_binary.c.

                                                                                      {
  int32_t retval;
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_ReadNonNativePageLastRows"))
    return (0);
  if (SDDS_dataset->layout.disconnected) {
    SDDS_SetError("Can't read page--file is disconnected (SDDS_ReadNonNativePageLastRows)");
    return 0;
  }
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!SDDS_dataset->layout.gzfp) {
      SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageLastRows)");
      return (0);
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!SDDS_dataset->layout.lzmafp) {
        SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageLastRows)");
        return (0);
      }
    } else {
      if (!SDDS_dataset->layout.fp) {
        SDDS_SetError("Unable to read page--NULL file pointer (SDDS_ReadNonNativePageLastRows)");
        return (0);
      }
    }
#if defined(zLib)
  }
#endif
  if (SDDS_dataset->original_layout.data_mode.mode == SDDS_ASCII) {
    if ((retval = SDDS_ReadAsciiPageLastRows(SDDS_dataset, last_rows)) < 1) {
      return (retval);
    }
  } else if (SDDS_dataset->original_layout.data_mode.mode == SDDS_BINARY) {
    if ((retval = SDDS_ReadNonNativeBinaryPageLastRows(SDDS_dataset, last_rows)) < 1) {
      return (retval);
    }
  } else {
    SDDS_SetError("Unable to read page--unrecognized data mode (SDDS_ReadNonNativePageLastRows)");
    return (0);
  }
  return (retval);
}

SDDS_ReadNonNativePageSparse()

int32_t SDDS_ReadNonNativePageSparse	(	SDDS_DATASET *	SDDS_dataset,
		uint32_t	mode,
		int64_t	sparse_interval,
		int64_t	sparse_offset )

Reads a sparse non-native endian page from an SDDS dataset.

This function reads a sparse page of data from the specified SDDS dataset, handling data with non-native endianness. Sparse reading allows for selective row retrieval based on the provided interval and offset.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to read from.
[in]	mode	Mode flag to support future expansion.
[in]	sparse_interval	Interval between rows to be read for sparsity.
[in]	sparse_offset	Offset to start reading rows for sparsity.

Returns

int32_t Returns the number of rows read on success, or 0 on failure.

Return values

>0	Number of rows successfully read.
0	An error occurred during the read operation, such as I/O failures, data corruption, or unsupported data modes.

Note

This function is a wrapper for SDDS_ReadNonNativePageDetailed with specific parameters to enable sparse reading. It should be used when selective row retrieval is desired.

Definition at line 4030 of file SDDS_binary.c.

                                                                                                                                {
  return SDDS_ReadNonNativePageDetailed(SDDS_dataset, mode, sparse_interval, sparse_offset, 0);
}

SDDS_ReadRecoveryPossible()

int32_t SDDS_ReadRecoveryPossible

(

SDDS_DATASET *

SDDS_dataset

)

Checks if any data in an SDDS page was recovered after an error was detected.

This function inspects the SDDS dataset to determine if any data recovery was possible following an error during data reading. It resets the recovery flag after checking.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset.

Returns

• Returns 1 if recovery was possible.
• Returns 0 if no recovery was performed or if recovery was not possible.

The function performs the following steps:

• Retrieves the current state of the readRecoveryPossible flag from the dataset.
• Resets the readRecoveryPossible flag to 0.
• Returns the original state of the readRecoveryPossible flag.

Note

• This function is typically used after attempting to recover from a read error to verify if any partial data was successfully recovered.
• The recovery flag is automatically managed by other functions within the SDDS library.

Definition at line 2054 of file SDDS_binary.c.

                                                              {
  int32_t returnValue;
 
  returnValue = SDDS_dataset->readRecoveryPossible;
  SDDS_dataset->readRecoveryPossible = 0;
  return returnValue;
}

SDDS_SetBufferedRead()

int32_t SDDS_SetBufferedRead

(

int32_t

dummy

)

Definition at line 49 of file SDDS_binary.c.

                                            {
  return 0;
}

SDDS_SetDefaultIOBufferSize()

int32_t SDDS_SetDefaultIOBufferSize

(

int32_t

newValue

)

Sets the default I/O buffer size used for file operations.

This function updates the global defaultIOBufferSize variable, which determines the size of the I/O buffer used for file read/write operations. The initial default is SDDS_FILEBUFFER_SIZE, which is 262144 bytes.

Parameters

newValue

The new default I/O buffer size in bytes. If newValue is negative, the function returns the current buffer size without changing it. If newValue is between 0 and 128 (inclusive), it is treated as 0, effectively disabling buffering.

Returns

The previous default I/O buffer size if newValue is greater than or equal to 0; otherwise, returns the current default buffer size without changing it.

Definition at line 66 of file SDDS_binary.c.

                                                      {
  int32_t previous;
  if (newValue < 0)
    return defaultIOBufferSize;
  if (newValue < 128) /* arbitrary limit */
    newValue = 0;
  previous = defaultIOBufferSize;
  defaultIOBufferSize = newValue;
  return previous;
}

SDDS_SetReadRecoveryMode()

void SDDS_SetReadRecoveryMode	(	SDDS_DATASET *	SDDS_dataset,
		int32_t	mode )

Sets the read recovery mode for an SDDS dataset.

This function configures whether read recovery is possible for the specified SDDS dataset. Enabling recovery allows the dataset to attempt to recover partial data in case of read errors.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset.
mode	Integer flag indicating the recovery mode: 0 to disable read recovery. 1 to enable read recovery.

The function updates the readRecoveryPossible flag within the dataset structure based on the provided mode parameter. This flag is later checked by other functions to determine whether to attempt data recovery after encountering read errors.

Note

• Enabling read recovery does not guarantee that all data can be recovered after an error.
• It is recommended to enable recovery only if partial data recovery is acceptable in your application.

Definition at line 2082 of file SDDS_binary.c.

                                                                        {
  SDDS_dataset->readRecoveryPossible = mode;
}

SDDS_SwapDouble()

void SDDS_SwapDouble

(

double *

data

)

Swaps the endianness of a double.

This function swaps the byte order of a 64-bit double-precision floating-point number pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the double whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned double.

Definition at line 3958 of file SDDS_binary.c.

                                   {
  double copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 7; i < 8; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapEndsArrayData()

int32_t SDDS_SwapEndsArrayData

(

SDDS_DATASET *

SDDSin

)

Swaps the endianness of the array data in an SDDS dataset.

This function iterates through all arrays defined in the specified SDDS dataset and swaps the byte order of each element to match the system's native endianness. It supports various data types including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. The function ensures that binary data is correctly interpreted on systems with different byte orders.

Parameters

[in,out]

SDDSin

Pointer to the SDDS_DATASET structure representing the dataset whose array data endianness is to be swapped.

Returns

int32_t Always returns 1.

Return values

1	The endianness of all applicable array data elements was successfully swapped.

Note

This function modifies the dataset's array data in place. It should be called only when the dataset's byte order is known to differ from the system's native byte order.

Definition at line 3747 of file SDDS_binary.c.

                                                     {
  int32_t i, j;
  SDDS_LAYOUT *layout;
  short *sData;
  unsigned short *suData;
  int32_t *lData;
  uint32_t *luData;
  int64_t *lData64;
  uint64_t *luData64;
  float *fData;
  double *dData;
  long double *ldData;
 
  layout = &SDDSin->layout;
 
  for (i = 0; i < layout->n_arrays; i++) {
    switch (layout->array_definition[i].type) {
    case SDDS_SHORT:
      sData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapShort(sData + j);
      break;
    case SDDS_USHORT:
      suData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapUShort(suData + j);
      break;
    case SDDS_LONG:
      lData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapLong(lData + j);
      break;
    case SDDS_ULONG:
      luData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapULong(luData + j);
      break;
    case SDDS_LONG64:
      lData64 = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapLong64(lData64 + j);
      break;
    case SDDS_ULONG64:
      luData64 = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapULong64(luData64 + j);
      break;
    case SDDS_LONGDOUBLE:
      ldData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapLongDouble(ldData + j);
      break;
    case SDDS_DOUBLE:
      dData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapDouble(dData + j);
      break;
    case SDDS_FLOAT:
      fData = SDDSin->array[i].data;
      for (j = 0; j < SDDSin->array[i].elements; j++)
        SDDS_SwapFloat(fData + j);
      break;
    default:
      break;
    }
  }
  return (1);
}

SDDS_SwapEndsColumnData()

int32_t SDDS_SwapEndsColumnData

(

SDDS_DATASET *

SDDSin

)

Swaps the endianness of the column data in an SDDS dataset.

This function iterates through all columns in the specified SDDS dataset and swaps the byte order of each data element to match the system's native endianness. It supports various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. The function ensures that binary data is correctly interpreted on systems with different byte orders.

Parameters

[in,out]

SDDSin

Pointer to the SDDS_DATASET structure representing the dataset whose column data endianness is to be swapped.

Returns

int32_t Always returns 1.

Return values

1	The endianness of all applicable column data elements was successfully swapped.

Note

This function modifies the dataset's column data in place. It should be called only when the dataset's byte order is known to differ from the system's native byte order. String data types are not affected by this function.

Definition at line 3582 of file SDDS_binary.c.

                                                      {
  int32_t i, row;
  SDDS_LAYOUT *layout;
  short *sData;
  unsigned short *suData;
  int32_t *lData;
  uint32_t *luData;
  int64_t *lData64;
  uint64_t *luData64;
  float *fData;
  double *dData;
  long double *ldData;
 
  layout = &SDDSin->layout;
  for (i = 0; i < layout->n_columns; i++) {
    switch (layout->column_definition[i].type) {
    case SDDS_SHORT:
      sData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapShort(sData + row);
      break;
    case SDDS_USHORT:
      suData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapUShort(suData + row);
      break;
    case SDDS_LONG:
      lData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapLong(lData + row);
      break;
    case SDDS_ULONG:
      luData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapULong(luData + row);
      break;
    case SDDS_LONG64:
      lData64 = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapLong64(lData64 + row);
      break;
    case SDDS_ULONG64:
      luData64 = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapULong64(luData64 + row);
      break;
    case SDDS_LONGDOUBLE:
      ldData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapLongDouble(ldData + row);
      break;
    case SDDS_DOUBLE:
      dData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapDouble(dData + row);
      break;
    case SDDS_FLOAT:
      fData = SDDSin->data[i];
      for (row = 0; row < SDDSin->n_rows; row++)
        SDDS_SwapFloat(fData + row);
      break;
    default:
      break;
    }
  }
  return (1);
}

SDDS_SwapEndsParameterData()

int32_t SDDS_SwapEndsParameterData

(

SDDS_DATASET *

SDDSin

)

Swaps the endianness of the parameter data in an SDDS dataset.

This function iterates through all parameters in the specified SDDS dataset and swaps the byte order of each data element to match the system's native endianness. It handles various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. Parameters with fixed values are skipped as their byte order is already consistent.

Parameters

[in,out]

SDDSin

Pointer to the SDDS_DATASET structure representing the dataset whose parameter data endianness is to be swapped.

Returns

int32_t Always returns 1.

Return values

1	The endianness of all applicable parameter data elements was successfully swapped.

Note

This function modifies the dataset's parameter data in place. It should be called only when the dataset's byte order is known to differ from the system's native byte order. String data types and parameters with fixed values are not affected by this function.

Definition at line 3668 of file SDDS_binary.c.

                                                         {
  int32_t i;
  SDDS_LAYOUT *layout;
  short *sData;
  unsigned short *suData;
  int32_t *lData;
  uint32_t *luData;
  int64_t *lData64;
  uint64_t *luData64;
  float *fData;
  double *dData;
  long double *ldData;
 
  layout = &SDDSin->layout;
  for (i = 0; i < layout->n_parameters; i++) {
    if (layout->parameter_definition[i].fixed_value) {
      continue;
    }
    switch (layout->parameter_definition[i].type) {
    case SDDS_SHORT:
      sData = SDDSin->parameter[i];
      SDDS_SwapShort(sData);
      break;
    case SDDS_USHORT:
      suData = SDDSin->parameter[i];
      SDDS_SwapUShort(suData);
      break;
    case SDDS_LONG:
      lData = SDDSin->parameter[i];
      SDDS_SwapLong(lData);
      break;
    case SDDS_ULONG:
      luData = SDDSin->parameter[i];
      SDDS_SwapULong(luData);
      break;
    case SDDS_LONG64:
      lData64 = SDDSin->parameter[i];
      SDDS_SwapLong64(lData64);
      break;
    case SDDS_ULONG64:
      luData64 = SDDSin->parameter[i];
      SDDS_SwapULong64(luData64);
      break;
    case SDDS_LONGDOUBLE:
      ldData = SDDSin->parameter[i];
      SDDS_SwapLongDouble(ldData);
      break;
    case SDDS_DOUBLE:
      dData = SDDSin->parameter[i];
      SDDS_SwapDouble(dData);
      break;
    case SDDS_FLOAT:
      fData = SDDSin->parameter[i];
      SDDS_SwapFloat(fData);
      break;
    default:
      break;
    }
  }
  return (1);
}

SDDS_SwapFloat()

void SDDS_SwapFloat

(

float *

data

)

Swaps the endianness of a float.

This function swaps the byte order of a 32-bit floating-point number pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the float whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned float.

Definition at line 3939 of file SDDS_binary.c.

                                 {
  float copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 3; i < 4; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapLong()

void SDDS_SwapLong

(

int32_t *

data

)

Swaps the endianness of a 32-bit integer.

This function swaps the byte order of a 32-bit integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the 32-bit integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 32-bit integer.

Definition at line 3863 of file SDDS_binary.c.

                                  {
  int32_t copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 3; i < 4; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapLong64()

void SDDS_SwapLong64

(

int64_t *

data

)

Swaps the endianness of a 64-bit integer.

This function swaps the byte order of a 64-bit integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the 64-bit integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 64-bit integer.

Definition at line 3901 of file SDDS_binary.c.

                                    {
  int64_t copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 7; i < 8; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapLongDouble()

void SDDS_SwapLongDouble

(

long double *

data

)

Swaps the endianness of a long double.

This function swaps the byte order of a long double floating-point number pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats. The function accounts for different sizes of long double based on the system's architecture.

Parameters

[in,out]

data

Pointer to the long double whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned long double. The size of long double may vary between different systems.

Definition at line 3978 of file SDDS_binary.c.

                                            {
  long double copy;
  short i, j;
  copy = *data;
  if (LDBL_DIG == 18) {
    for (i = 0, j = 11; i < 12; i++, j--)
      *(((char *)data) + i) = *(((char *)&copy) + j);
  } else {
    for (i = 0, j = 7; i < 8; i++, j--)
      *(((char *)data) + i) = *(((char *)&copy) + j);
  }
}

SDDS_SwapShort()

void SDDS_SwapShort

(

short *

data

)

Swaps the endianness of a short integer.

This function swaps the byte order of a 16-bit short integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the short integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 16-bit short integer.

Definition at line 3827 of file SDDS_binary.c.

                                 {
  unsigned char c1;
  c1 = *((char *)data);
  *((char *)data) = *(((char *)data) + 1);
  *(((char *)data) + 1) = c1;
}

SDDS_SwapULong()

void SDDS_SwapULong

(

uint32_t *

data

)

Swaps the endianness of a 32-bit unsigned integer.

This function swaps the byte order of a 32-bit unsigned integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the 32-bit unsigned integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 32-bit unsigned integer.

Definition at line 3882 of file SDDS_binary.c.

                                    {
  uint32_t copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 3; i < 4; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapULong64()

void SDDS_SwapULong64

(

uint64_t *

data

)

Swaps the endianness of a 64-bit unsigned integer.

This function swaps the byte order of a 64-bit unsigned integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the 64-bit unsigned integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 64-bit unsigned integer.

Definition at line 3920 of file SDDS_binary.c.

                                      {
  uint64_t copy;
  short i, j;
  copy = *data;
  for (i = 0, j = 7; i < 8; i++, j--)
    *(((char *)data) + i) = *(((char *)&copy) + j);
}

SDDS_SwapUShort()

void SDDS_SwapUShort

(

unsigned short *

data

)

Swaps the endianness of an unsigned short integer.

This function swaps the byte order of a 16-bit unsigned short integer pointed to by the provided data pointer. It effectively converts the data between little-endian and big-endian formats.

Parameters

[in,out]

data

Pointer to the unsigned short integer whose byte order is to be swapped.

Note

The function modifies the data in place. Ensure that the pointer is valid and points to a properly aligned 16-bit unsigned short integer.

Definition at line 3845 of file SDDS_binary.c.

                                           {
  unsigned char c1;
  c1 = *((char *)data);
  *((char *)data) = *(((char *)data) + 1);
  *(((char *)data) + 1) = c1;
}

SDDS_UpdateBinaryPage()

int32_t SDDS_UpdateBinaryPage	(	SDDS_DATASET *	SDDS_dataset,
		uint32_t	mode )

Updates the binary page of an SDDS dataset.

This function updates the binary page of the specified SDDS dataset based on the provided mode. It handles writing the dataset's binary data to the associated file, managing buffering, and handling different file formats such as gzip and LZMA if applicable.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to update.
mode	Bitmask indicating the update mode. It can be: 0 for a standard update. FLUSH_TABLE to flush the table after updating.

Returns

• Returns 1 on successful update.
• Returns 0 if an error occurs during the update process.

The function performs several checks before updating:

• Checks the environment variable SDDS_OUTPUT_ENDIANESS to determine if a non-native binary update is required.
• Validates the dataset structure.
• Ensures that the dataset is not using gzip or LZMA compression, or is not in column-major data mode.
• Handles writing the binary page, updating row counts, and managing buffer flushing.

Note

• The function is not thread-safe and should be called in a synchronized context.
• Requires that the dataset has been properly initialized and populated with data.

Definition at line 1077 of file SDDS_binary.c.

                                                                         {
  FILE *fp;
  int64_t i, rows, offset, code, fixed_rows;
  int32_t min32 = INT32_MIN, rows32;
  SDDS_FILEBUFFER *fBuffer;
  char *outputEndianess = NULL;
 
  if ((outputEndianess = getenv("SDDS_OUTPUT_ENDIANESS"))) {
    if (((strncmp(outputEndianess, "big", 3) == 0) && (SDDS_IsBigEndianMachine() == 0)) || ((strncmp(outputEndianess, "little", 6) == 0) && (SDDS_IsBigEndianMachine() == 1)))
      return SDDS_UpdateNonNativeBinaryPage(SDDS_dataset, mode);
  }
 
#ifdef DEBUG
  fprintf(stderr, "%" PRId64 " virtual rows present, first=%" PRId64 "\n", SDDS_CountRowsOfInterest(SDDS_dataset), SDDS_dataset->first_row_in_mem);
#endif
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_UpdateBinaryPage"))
    return (0);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    SDDS_SetError("Unable to perform page updates on a gzip file (SDDS_UpdateBinaryPage)");
    return 0;
  }
#endif
  if (SDDS_dataset->layout.lzmaFile) {
    SDDS_SetError("Unable to perform page updates on an .lzma or .xz file (SDDS_UpdateBinaryPage)");
    return 0;
  }
  if (SDDS_dataset->layout.data_mode.column_major) {
    SDDS_SetError("Unable to perform page updates on column major order file. (SDDS_UpdateBinaryPage)");
    return 0;
  }
  if (!SDDS_dataset->writing_page) {
#ifdef DEBUG
    fprintf(stderr, "Page not being written---calling SDDS_UpdateBinaryPage\n");
#endif
    if (!(code = SDDS_WriteBinaryPage(SDDS_dataset)))
      return 0;
    if (mode & FLUSH_TABLE) {
      SDDS_FreeTableStrings(SDDS_dataset);
      SDDS_dataset->first_row_in_mem = SDDS_CountRowsOfInterest(SDDS_dataset);
      SDDS_dataset->last_row_written = -1;
      SDDS_dataset->n_rows = 0;
    }
    return code;
  }
 
  if (!(fp = SDDS_dataset->layout.fp)) {
    SDDS_SetError("Unable to update page--file pointer is NULL (SDDS_UpdateBinaryPage)");
    return (0);
  }
  fBuffer = &SDDS_dataset->fBuffer;
  if (!SDDS_FlushBuffer(fp, fBuffer)) {
    SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_UpdateBinaryPage)");
    return 0;
  }
  offset = ftell(fp);
 
  rows = SDDS_CountRowsOfInterest(SDDS_dataset) + SDDS_dataset->first_row_in_mem;
#ifdef DEBUG
  fprintf(stderr, "%" PRId64 " rows stored in table, %" PRId64 " already written\n", rows, SDDS_dataset->n_rows_written);
#endif
  if (rows == SDDS_dataset->n_rows_written)
    return (1);
  if (rows < SDDS_dataset->n_rows_written) {
    SDDS_SetError("Unable to update page--new number of rows less than previous number (SDDS_UpdateBinaryPage)");
    return (0);
  }
  if ((!SDDS_dataset->layout.data_mode.fixed_row_count) || (((rows + rows - SDDS_dataset->n_rows_written) / SDDS_dataset->layout.data_mode.fixed_row_increment) != (rows / SDDS_dataset->layout.data_mode.fixed_row_increment))) {
    if (SDDS_fseek(fp, SDDS_dataset->rowcount_offset, 0) == -1) {
      SDDS_SetError("Unable to update page--failure doing fseek (SDDS_UpdateBinaryPage)");
      return (0);
    }
    if (SDDS_dataset->layout.data_mode.fixed_row_count) {
      if ((rows - SDDS_dataset->n_rows_written) + 1 > SDDS_dataset->layout.data_mode.fixed_row_increment) {
        SDDS_dataset->layout.data_mode.fixed_row_increment = (rows - SDDS_dataset->n_rows_written) + 1;
      }
      fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
#if defined(DEBUG)
      fprintf(stderr, "Setting %" PRId64 " fixed rows\n", fixed_rows);
#endif
      if ((fixed_rows > INT32_MAX) && (SDDS_dataset->n_rows_written <= INT32_MAX)) {
        SDDS_SetError("Unable to update page--crossed the INT32_MAX row boundary (SDDS_UpdateBinaryPage)");
        return (0);
      }
      if (fixed_rows > INT32_MAX) {
        if (fwrite(&min32, sizeof(min32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
        if (fwrite(&fixed_rows, sizeof(fixed_rows), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
      } else {
        rows32 = (int32_t)fixed_rows;
        if (fwrite(&fixed_rows, sizeof(rows32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
      }
    } else {
#if defined(DEBUG)
      fprintf(stderr, "Setting %" PRId64 " rows\n", rows);
#endif
      if ((rows > INT32_MAX) && (SDDS_dataset->n_rows_written <= INT32_MAX)) {
        SDDS_SetError("Unable to update page--crossed the INT32_MAX row boundary (SDDS_UpdateBinaryPage)");
        return (0);
      }
      if (rows > INT32_MAX) {
        if (fwrite(&min32, sizeof(min32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
        if (fwrite(&rows, sizeof(rows), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
      } else {
        rows32 = (int32_t)rows;
        if (fwrite(&rows32, sizeof(rows32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
      }
    }
    if (SDDS_fseek(fp, offset, 0) == -1) {
      SDDS_SetError("Unable to update page--failure doing fseek to end of page (SDDS_UpdateBinaryPage)");
      return (0);
    }
  }
  for (i = SDDS_dataset->last_row_written + 1; i < SDDS_dataset->n_rows; i++)
    if (SDDS_dataset->row_flag[i] && !SDDS_WriteBinaryRow(SDDS_dataset, i)) {
      SDDS_SetError("Unable to update page--failure writing row (SDDS_UpdateBinaryPage)");
      return (0);
    }
#ifdef DEBUG
  fprintf(stderr, "Flushing buffer\n");
#endif
  if (!SDDS_FlushBuffer(fp, fBuffer)) {
    SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_UpdateBinaryPage)");
    return 0;
  }
  SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
  SDDS_dataset->n_rows_written = rows;
  if (mode & FLUSH_TABLE) {
    SDDS_FreeTableStrings(SDDS_dataset);
    SDDS_dataset->first_row_in_mem = rows;
    SDDS_dataset->last_row_written = -1;
    SDDS_dataset->n_rows = 0;
  }
  return (1);
}

SDDS_UpdateNonNativeBinaryPage()

int32_t SDDS_UpdateNonNativeBinaryPage	(	SDDS_DATASET *	SDDS_dataset,
		uint32_t	mode )

Updates a non-native endian binary page in an SDDS dataset.

This function updates an existing binary page in the specified SDDS dataset, handling byte order reversal to convert between little-endian and big-endian formats. It supports updating rows based on the provided mode flags, such as flushing the table. The function ensures that the buffer is flushed before performing the update and writes any new rows that have been flagged for writing. It also handles fixed row counts and manages byte order conversions as necessary.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to update.
[in]	mode	Bitmask indicating the update mode (e.g., FLUSH_TABLE).

Returns

int32_t Returns 1 on successful update of the binary page, or 0 if an error occurred.

Return values

1	The binary page was successfully updated and byte-swapped.
0	An error occurred during the update operation, such as I/O failures, invalid row counts, or corrupted dataset definitions.

Note

This function modifies the dataset's internal structures during the update process. Ensure that the dataset is properly initialized and opened for writing before invoking this function. After updating, the dataset's state is updated to reflect the changes made to the page.

Definition at line 5717 of file SDDS_binary.c.

                                                                                  {
  FILE *fp;
  int32_t code, min32 = INT32_MIN, rows32;
  int64_t i, rows, offset, fixed_rows;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_UpdateNonNativeBinaryPage"))
    return (0);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    SDDS_SetError("Unable to perform page updates on a gzip file (SDDS_UpdateNonNativeBinaryPage)");
    return 0;
  }
#endif
  if (SDDS_dataset->layout.lzmaFile) {
    SDDS_SetError("Unable to perform page updates on .lzma or .xz files (SDDS_UpdateNonNativeBinaryPage)");
    return 0;
  }
  if (SDDS_dataset->layout.data_mode.column_major) {
    SDDS_SetError("Unable to perform page updates on a column major order file (SDDS_UpdateNonNativeBinaryPage)");
    return 0;
  }
  if (!SDDS_dataset->writing_page) {
#ifdef DEBUG
    fprintf(stderr, "Page not being written---calling SDDS_UpdateNonNativeBinaryPage\n");
#endif
    if (!(code = SDDS_WriteNonNativeBinaryPage(SDDS_dataset))) {
      return 0;
    }
    if (mode & FLUSH_TABLE) {
      SDDS_FreeTableStrings(SDDS_dataset);
      SDDS_dataset->first_row_in_mem = SDDS_CountRowsOfInterest(SDDS_dataset);
      SDDS_dataset->last_row_written = -1;
      SDDS_dataset->n_rows = 0;
    }
    return code;
  }
 
  if (!(fp = SDDS_dataset->layout.fp)) {
    SDDS_SetError("Unable to update page--file pointer is NULL (SDDS_UpdateNonNativeBinaryPage)");
    return (0);
  }
  fBuffer = &SDDS_dataset->fBuffer;
  if (!SDDS_FlushBuffer(fp, fBuffer)) {
    SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_UpdateNonNativeBinaryPage)");
    return 0;
  }
  offset = ftell(fp);
 
  rows = SDDS_CountRowsOfInterest(SDDS_dataset) + SDDS_dataset->first_row_in_mem;
#ifdef DEBUG
  fprintf(stderr, "%" PRId64 " rows stored in table, %" PRId32 " already written\n", rows, SDDS_dataset->n_rows_written);
#endif
  if (rows == SDDS_dataset->n_rows_written) {
    return (1);
  }
  if (rows < SDDS_dataset->n_rows_written) {
    SDDS_SetError("Unable to update page--new number of rows less than previous number (SDDS_UpdateNonNativeBinaryPage)");
    return (0);
  }
  SDDS_SwapLong(&min32);
  if ((!SDDS_dataset->layout.data_mode.fixed_row_count) || (((rows + rows - SDDS_dataset->n_rows_written / SDDS_dataset->layout.data_mode.fixed_row_increment)) != (rows / SDDS_dataset->layout.data_mode.fixed_row_increment))) {
    if (SDDS_fseek(fp, SDDS_dataset->rowcount_offset, 0) == -1) {
      SDDS_SetError("Unable to update page--failure doing fseek (SDDS_UpdateNonNativeBinaryPage)");
      return (0);
    }
    if (SDDS_dataset->layout.data_mode.fixed_row_count) {
      if ((rows - SDDS_dataset->n_rows_written) + 1 > SDDS_dataset->layout.data_mode.fixed_row_increment) {
        SDDS_dataset->layout.data_mode.fixed_row_increment = (rows - SDDS_dataset->n_rows_written) + 1;
      }
      fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
#if defined(DEBUG)
      fprintf(stderr, "Setting %" PRId64 " fixed rows\n", fixed_rows);
#endif
      if ((fixed_rows > INT32_MAX) && (SDDS_dataset->n_rows_written <= INT32_MAX)) {
        SDDS_SetError("Unable to update page--crossed the INT32_MAX row boundary (SDDS_UpdateNonNativeBinaryPage)");
        return (0);
      }
      if (fixed_rows > INT32_MAX) {
        if (fwrite(&min32, sizeof(min32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&fixed_rows);
        if (fwrite(&fixed_rows, sizeof(fixed_rows), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&fixed_rows);
      } else {
        rows32 = (int32_t)fixed_rows;
        SDDS_SwapLong(&rows32);
        if (fwrite(&rows32, sizeof(rows32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateNonNativeBinaryPage)");
          return (0);
        }
      }
    } else {
#if defined(DEBUG)
      fprintf(stderr, "Setting %" PRId64 " rows\n", rows);
#endif
      if ((rows > INT32_MAX) && (SDDS_dataset->n_rows_written <= INT32_MAX)) {
        SDDS_SetError("Unable to update page--crossed the INT32_MAX row boundary (SDDS_UpdateNonNativeBinaryPage)");
        return (0);
      }
      if (rows > INT32_MAX) {
        if (fwrite(&min32, sizeof(min32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&rows);
        if (fwrite(&rows, sizeof(rows), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&rows);
      } else {
        rows32 = (int32_t)rows;
        SDDS_SwapLong(&rows32);
        if (fwrite(&rows32, sizeof(rows32), 1, fp) != 1) {
          SDDS_SetError("Unable to update page--failure writing number of rows (SDDS_UpdateNonNativeBinaryPage)");
          return (0);
        }
      }
    }
    if (SDDS_fseek(fp, offset, 0) == -1) {
      SDDS_SetError("Unable to update page--failure doing fseek to end of page (SDDS_UpdateNonNativeBinaryPage)");
      return (0);
    }
  }
  SDDS_SwapEndsColumnData(SDDS_dataset);
  for (i = SDDS_dataset->last_row_written + 1; i < SDDS_dataset->n_rows; i++) {
    if (SDDS_dataset->row_flag[i] && !SDDS_WriteNonNativeBinaryRow(SDDS_dataset, i)) {
      SDDS_SetError("Unable to update page--failure writing row (SDDS_UpdateNonNativeBinaryPage)");
      return (0);
    }
  }
  SDDS_SwapEndsColumnData(SDDS_dataset);
#ifdef DEBUG
  fprintf(stderr, "Flushing buffer\n");
#endif
  if (!SDDS_FlushBuffer(fp, fBuffer)) {
    SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_UpdateNonNativeBinaryPage)");
    return 0;
  }
  SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
  SDDS_dataset->n_rows_written = rows;
  if (mode & FLUSH_TABLE) {
    SDDS_FreeTableStrings(SDDS_dataset);
    SDDS_dataset->first_row_in_mem = rows;
    SDDS_dataset->last_row_written = -1;
    SDDS_dataset->n_rows = 0;
  }
  return (1);
}

SDDS_WriteBinaryArrays()

int32_t SDDS_WriteBinaryArrays

(

SDDS_DATASET *

SDDS_dataset

)

Writes the binary arrays of the SDDS dataset to a file.

This function writes all arrays defined in the SDDS dataset to the associated binary file. It handles arrays with and without dimensions, and manages different compression formats such as gzip and LZMA if enabled.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure containing the dataset to write.

Returns

• Returns 1 on successful write of all arrays.
• Returns 0 if an error occurs during the writing process.

The function performs the following steps:

• Validates the dataset structure.
• Iterates over all arrays defined in the dataset layout.
• For each array:
  • If the array has no dimensions, it writes zeroes for each defined dimension.
  • If the array has dimensions, it writes the dimension sizes using the appropriate compression method.
  • If the array type is SDDS_STRING, it writes each string element using the appropriate compression method.
  • Otherwise, it writes the array's data using buffered write functions.

The function handles different file formats:

• For gzip files, it uses SDDS_GZipWriteBinaryString and SDDS_GZipBufferedWrite.
• For LZMA files, it uses SDDS_LZMAWriteBinaryString and SDDS_LZMABufferedWrite.
• For standard binary files, it uses SDDS_WriteBinaryString and SDDS_BufferedWrite.

Note

• The function assumes that the dataset has been properly initialized and that all arrays are correctly allocated.
• Compression support (zLib or LZMA) must be enabled during compilation for handling compressed files.

Definition at line 1533 of file SDDS_binary.c.

                                                           {
  int32_t i, j, zero = 0;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteBinaryArrays"))
    return (0);
  layout = &SDDS_dataset->layout;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    fBuffer = &SDDS_dataset->fBuffer;
    for (i = 0; i < layout->n_arrays; i++) {
      if (!SDDS_dataset->array[i].dimension) {
        for (j = 0; j < layout->array_definition[i].dimensions; j++)
          if (!SDDS_GZipBufferedWrite(&zero, sizeof(zero), gzfp, fBuffer)) {
            SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteBinaryArrays)");
            return 0;
          }
        continue;
      }
      if (!SDDS_GZipBufferedWrite(SDDS_dataset->array[i].dimension, sizeof(*(SDDS_dataset->array)[i].dimension) * layout->array_definition[i].dimensions, gzfp, fBuffer)) {
        SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteBinaryArrays)");
        return (0);
      }
      if (layout->array_definition[i].type == SDDS_STRING) {
        for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
          if (!SDDS_GZipWriteBinaryString(((char **)SDDS_dataset->array[i].data)[j], gzfp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryArrays)");
            return (0);
          }
        }
      } else if (!SDDS_GZipBufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, gzfp, fBuffer)) {
        SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteBinaryArrays)");
        return (0);
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.gzipFile) {
      lzmafp = layout->lzmafp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_arrays; i++) {
        if (!SDDS_dataset->array[i].dimension) {
          for (j = 0; j < layout->array_definition[i].dimensions; j++)
            if (!SDDS_LZMABufferedWrite(&zero, sizeof(zero), lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteBinaryArrays)");
              return 0;
            }
          continue;
        }
        if (!SDDS_LZMABufferedWrite(SDDS_dataset->array[i].dimension, sizeof(*(SDDS_dataset->array)[i].dimension) * layout->array_definition[i].dimensions, lzmafp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteBinaryArrays)");
          return (0);
        }
        if (layout->array_definition[i].type == SDDS_STRING) {
          for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
            if (!SDDS_LZMAWriteBinaryString(((char **)SDDS_dataset->array[i].data)[j], lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryArrays)");
              return (0);
            }
          }
        } else if (!SDDS_LZMABufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, lzmafp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteBinaryArrays)");
          return (0);
        }
      }
    } else {
      fp = layout->fp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_arrays; i++) {
        if (!SDDS_dataset->array[i].dimension) {
          for (j = 0; j < layout->array_definition[i].dimensions; j++)
            if (!SDDS_BufferedWrite(&zero, sizeof(zero), fp, fBuffer)) {
              SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteBinaryArrays)");
              return 0;
            }
          continue;
        }
        if (!SDDS_BufferedWrite(SDDS_dataset->array[i].dimension, sizeof(*(SDDS_dataset->array)[i].dimension) * layout->array_definition[i].dimensions, fp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteBinaryArrays)");
          return (0);
        }
        if (layout->array_definition[i].type == SDDS_STRING) {
          for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
            if (!SDDS_WriteBinaryString(((char **)SDDS_dataset->array[i].data)[j], fp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryArrays)");
              return (0);
            }
          }
        } else if (!SDDS_BufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, fp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteBinaryArrays)");
          return (0);
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteBinaryColumns()

int32_t SDDS_WriteBinaryColumns

(

SDDS_DATASET *

SDDS_dataset

)

Writes the binary columns of an SDDS dataset to the associated file.

This function iterates over each column defined in the SDDS dataset layout and writes its data to the binary file. It handles different data types, including strings and numeric types, and supports various compression formats such as gzip and LZMA if enabled.

Depending on the dataset's configuration, the function writes directly to a standard binary file, a gzip-compressed file, or an LZMA-compressed file. It also handles sparse data by only writing rows flagged for inclusion.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to write.

Returns

• Returns 1 on successful writing of all columns.
• Returns 0 if an error occurs during the writing process.

The function performs the following steps:

• Validates the dataset structure using SDDS_CheckDataset.
• Determines the file format (standard, gzip, LZMA) and initializes the corresponding file pointer.
• Iterates through each column in the dataset:
  • For string columns, writes each string entry individually using the appropriate write function.
  • For numeric columns, writes the entire column data in a buffered manner if all rows are flagged; otherwise, writes individual row entries.
• Handles errors by setting appropriate error messages and aborting the write operation.

Note

• The function assumes that the dataset has been properly initialized and populated with data.
• Compression support (zLib for gzip, LZMA libraries) must be enabled during compilation for handling compressed files.
• The function is not thread-safe and should be called in a synchronized context.

Definition at line 1673 of file SDDS_binary.c.

                                                            {
  int64_t i, row, rows, type, size;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteBinaryColumns"))
    return (0);
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
  rows = SDDS_CountRowsOfInterest(SDDS_dataset);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_columns; i++) {
      type = layout->column_definition[i].type;
      size = SDDS_type_size[type - 1];
      if (type == SDDS_STRING) {
        for (row = 0; row < SDDS_dataset->n_rows; row++) {
          if (SDDS_dataset->row_flag[row] && !SDDS_GZipWriteBinaryString(*((char **)SDDS_dataset->data[i] + row), gzfp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryColumns)");
            return (0);
          }
        }
      } else {
        if (rows == SDDS_dataset->n_rows) {
          if (!SDDS_GZipBufferedWrite(SDDS_dataset->data[i], size * rows, gzfp, fBuffer)) {
            SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
            return (0);
          }
        } else {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_GZipBufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
              return (0);
            }
          }
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_columns; i++) {
        type = layout->column_definition[i].type;
        size = SDDS_type_size[type - 1];
        if (layout->column_definition[i].type == SDDS_STRING) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_LZMAWriteBinaryString(*((char **)SDDS_dataset->data[i] + row), lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryColumns)");
              return (0);
            }
          }
        } else {
          if (rows == SDDS_dataset->n_rows) {
            if (!SDDS_LZMABufferedWrite(SDDS_dataset->data[i], size * rows, lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
              return (0);
            }
          } else {
            for (row = 0; row < SDDS_dataset->n_rows; row++) {
              if (SDDS_dataset->row_flag[row] && !SDDS_LZMABufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer)) {
                SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
                return (0);
              }
            }
          }
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_columns; i++) {
        type = layout->column_definition[i].type;
        size = SDDS_type_size[type - 1];
        if (layout->column_definition[i].type == SDDS_STRING) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_WriteBinaryString(*((char **)SDDS_dataset->data[i] + row), fp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteBinaryColumns)");
              return (0);
            }
          }
        } else {
          if (rows == SDDS_dataset->n_rows) {
            if (!SDDS_BufferedWrite(SDDS_dataset->data[i], size * rows, fp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
              return (0);
            }
          } else {
            for (row = 0; row < SDDS_dataset->n_rows; row++) {
              if (SDDS_dataset->row_flag[row] && !SDDS_BufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer)) {
                SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteBinaryColumns)");
                return (0);
              }
            }
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteBinaryPage()

int32_t SDDS_WriteBinaryPage

(

SDDS_DATASET *

SDDS_dataset

)

Writes a binary page of data to an SDDS dataset, handling compression and buffering.

This function writes a binary page (including parameters, arrays, and row data) to the SDDS dataset pointed to by SDDS_dataset. It handles different file types, including regular files, LZMA-compressed files, and GZIP-compressed files, and uses buffering to improve write performance.

The function performs the following steps:

• Checks for output endianess and writes a non-native binary page if needed.
• Determines the number of rows to write and calculates any fixed row counts.
• Writes the number of rows to the file.
• Writes parameters, arrays, and column data using the appropriate write functions.
• Flushes the buffer to ensure all data is written.

It uses the appropriate buffered write functions (SDDS_BufferedWrite, SDDS_LZMABufferedWrite, or SDDS_GZipBufferedWrite) depending on the file type.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the SDDS dataset to write to.

Returns

Returns 1 on success; returns 0 on error.

Note

The function sets error messages using SDDS_SetError if any step fails.

Definition at line 749 of file SDDS_binary.c.

                                                         {
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  int64_t i, rows, fixed_rows;
  int32_t min32 = INT32_MIN, rows32;
  /*  static char buffer[SDDS_MAXLINE]; */
  SDDS_FILEBUFFER *fBuffer;
  char *outputEndianess = NULL;
 
  if ((outputEndianess = getenv("SDDS_OUTPUT_ENDIANESS"))) {
    if (((strncmp(outputEndianess, "big", 3) == 0) && (SDDS_IsBigEndianMachine() == 0)) || ((strncmp(outputEndianess, "little", 6) == 0) && (SDDS_IsBigEndianMachine() == 1)))
      return SDDS_WriteNonNativeBinaryPage(SDDS_dataset);
  }
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteBinaryPage"))
    return (0);
 
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!(gzfp = SDDS_dataset->layout.gzfp)) {
      SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteBinaryPage)");
      return (0);
    }
    fBuffer = &SDDS_dataset->fBuffer;
 
    if (!fBuffer->buffer) {
      if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * (defaultIOBufferSize + 1)))) {
        SDDS_SetError("Unable to do buffered read--allocation failure (SDDS_WriteBinaryPage)");
        return 0;
      }
      fBuffer->bufferSize = defaultIOBufferSize;
      fBuffer->bytesLeft = defaultIOBufferSize;
    }
 
    rows = SDDS_CountRowsOfInterest(SDDS_dataset);
    SDDS_dataset->rowcount_offset = gztell(gzfp);
    if (SDDS_dataset->layout.data_mode.fixed_row_count) {
      fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
      if (fixed_rows > INT32_MAX) {
        if (!SDDS_GZipBufferedWrite(&min32, sizeof(min32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
        if (!SDDS_GZipBufferedWrite(&fixed_rows, sizeof(fixed_rows), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
      } else {
        rows32 = (int32_t)fixed_rows;
        if (!SDDS_GZipBufferedWrite(&rows32, sizeof(rows32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
      }
    } else {
      if (rows > INT32_MAX) {
        if (!SDDS_GZipBufferedWrite(&min32, sizeof(min32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
        if (!SDDS_GZipBufferedWrite(&rows, sizeof(rows), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
      } else {
        rows32 = (int32_t)rows;
        if (!SDDS_GZipBufferedWrite(&rows32, sizeof(rows32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
          return (0);
        }
      }
    }
    if (!SDDS_WriteBinaryParameters(SDDS_dataset)) {
      SDDS_SetError("Unable to write page--parameter writing problem (SDDS_WriteBinaryPage)");
      return 0;
    }
    if (!SDDS_WriteBinaryArrays(SDDS_dataset)) {
      SDDS_SetError("Unable to write page--array writing problem (SDDS_WriteBinaryPage)");
      return 0;
    }
    if (SDDS_dataset->layout.n_columns) {
      if (SDDS_dataset->layout.data_mode.column_major) {
        if (!SDDS_WriteBinaryColumns(SDDS_dataset)) {
          SDDS_SetError("Unable to write page--column writing problem (SDDS_WriteBinaryPage)");
          return 0;
        }
      } else {
        for (i = 0; i < SDDS_dataset->n_rows; i++) {
          if (SDDS_dataset->row_flag[i] && !SDDS_WriteBinaryRow(SDDS_dataset, i)) {
            SDDS_SetError("Unable to write page--row writing problem (SDDS_WriteBinaryPage)");
            return 0;
          }
        }
      }
    }
    if (!SDDS_GZipFlushBuffer(gzfp, fBuffer)) {
      SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteBinaryPage)");
      return 0;
    }
    SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
    SDDS_dataset->n_rows_written = rows;
    SDDS_dataset->writing_page = 1;
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!(lzmafp = SDDS_dataset->layout.lzmafp)) {
        SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteBinaryPage)");
        return (0);
      }
      fBuffer = &SDDS_dataset->fBuffer;
 
      if (!fBuffer->buffer) {
        if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * (defaultIOBufferSize + 1)))) {
          SDDS_SetError("Unable to do buffered read--allocation failure (SDDS_WriteBinaryPage)");
          return 0;
        }
        fBuffer->bufferSize = defaultIOBufferSize;
        fBuffer->bytesLeft = defaultIOBufferSize;
      }
      rows = SDDS_CountRowsOfInterest(SDDS_dataset);
      SDDS_dataset->rowcount_offset = lzma_tell(lzmafp);
      if (SDDS_dataset->layout.data_mode.fixed_row_count) {
        fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
        if (fixed_rows > INT32_MAX) {
          if (!SDDS_LZMABufferedWrite(&min32, sizeof(min32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
          if (!SDDS_LZMABufferedWrite(&fixed_rows, sizeof(fixed_rows), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        } else {
          rows32 = (int32_t)fixed_rows;
          if (!SDDS_LZMABufferedWrite(&rows32, sizeof(rows32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        }
      } else {
        if (rows > INT32_MAX) {
          if (!SDDS_LZMABufferedWrite(&min32, sizeof(min32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
          if (!SDDS_LZMABufferedWrite(&rows, sizeof(rows), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        } else {
          rows32 = (int32_t)rows;
          if (!SDDS_LZMABufferedWrite(&rows32, sizeof(rows32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        }
      }
      if (!SDDS_WriteBinaryParameters(SDDS_dataset)) {
        SDDS_SetError("Unable to write page--parameter writing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      if (!SDDS_WriteBinaryArrays(SDDS_dataset)) {
        SDDS_SetError("Unable to write page--array writing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      if (SDDS_dataset->layout.n_columns) {
        if (SDDS_dataset->layout.data_mode.column_major) {
          if (!SDDS_WriteBinaryColumns(SDDS_dataset)) {
            SDDS_SetError("Unable to write page--column writing problem (SDDS_WriteBinaryPage)");
            return 0;
          }
        } else {
          for (i = 0; i < SDDS_dataset->n_rows; i++) {
            if (SDDS_dataset->row_flag[i] && !SDDS_WriteBinaryRow(SDDS_dataset, i)) {
              SDDS_SetError("Unable to write page--row writing problem (SDDS_WriteBinaryPage)");
              return 0;
            }
          }
        }
      }
      if (!SDDS_LZMAFlushBuffer(lzmafp, fBuffer)) {
        SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
      SDDS_dataset->n_rows_written = rows;
      SDDS_dataset->writing_page = 1;
    } else {
      if (!(fp = SDDS_dataset->layout.fp)) {
        SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteBinaryPage)");
        return (0);
      }
      fBuffer = &SDDS_dataset->fBuffer;
 
      if (!fBuffer->buffer) {
        if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * (defaultIOBufferSize + 1)))) {
          SDDS_SetError("Unable to do buffered read--allocation failure (SDDS_WriteBinaryPage)");
          return 0;
        }
        fBuffer->bufferSize = defaultIOBufferSize;
        fBuffer->bytesLeft = defaultIOBufferSize;
      }
 
      /* Flush any existing data in the output buffer so we can determine the
       * row count offset for the file.  This is probably unnecessary.
       */
      if (!SDDS_FlushBuffer(fp, fBuffer)) {
        SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
 
      /* output the row count and determine its byte offset in the file */
      rows = SDDS_CountRowsOfInterest(SDDS_dataset);
      SDDS_dataset->rowcount_offset = ftell(fp);
      if (SDDS_dataset->layout.data_mode.fixed_row_count) {
        fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
#if defined(DEBUG)
        fprintf(stderr, "setting %" PRId64 " fixed rows\n", fixed_rows);
#endif
        if (fixed_rows > INT32_MAX) {
          if (!SDDS_BufferedWrite(&min32, sizeof(min32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
          if (!SDDS_BufferedWrite(&fixed_rows, sizeof(fixed_rows), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        } else {
          rows32 = (int32_t)fixed_rows;
          if (!SDDS_BufferedWrite(&rows32, sizeof(rows32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        }
      } else {
#if defined(DEBUG)
        fprintf(stderr, "setting %" PRId64 " rows\n", rows);
#endif
        if (rows > INT32_MAX) {
          if (!SDDS_BufferedWrite(&min32, sizeof(min32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
          if (!SDDS_BufferedWrite(&rows, sizeof(rows), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        } else {
          rows32 = (int32_t)rows;
          if (!SDDS_BufferedWrite(&rows32, sizeof(rows32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteBinaryPage)");
            return (0);
          }
        }
      }
 
      /* write the data, using buffered I/O */
      if (!SDDS_WriteBinaryParameters(SDDS_dataset)) {
        SDDS_SetError("Unable to write page--parameter writing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      if (!SDDS_WriteBinaryArrays(SDDS_dataset)) {
        SDDS_SetError("Unable to write page--array writing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      if (SDDS_dataset->layout.n_columns) {
        if (SDDS_dataset->layout.data_mode.column_major) {
          if (!SDDS_WriteBinaryColumns(SDDS_dataset)) {
            SDDS_SetError("Unable to write page--column writing problem (SDDS_WriteBinaryPage)");
            return 0;
          }
        } else {
          for (i = 0; i < SDDS_dataset->n_rows; i++) {
            if (SDDS_dataset->row_flag[i] && !SDDS_WriteBinaryRow(SDDS_dataset, i)) {
              SDDS_SetError("Unable to write page--row writing problem (SDDS_WriteBinaryPage)");
              return 0;
            }
          }
        }
      }
      /* flush the page */
      if (!SDDS_FlushBuffer(fp, fBuffer)) {
        SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteBinaryPage)");
        return 0;
      }
      SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
      SDDS_dataset->n_rows_written = rows;
      SDDS_dataset->writing_page = 1;
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteBinaryParameters()

int32_t SDDS_WriteBinaryParameters

(

SDDS_DATASET *

SDDS_dataset

)

Writes the binary parameters of the SDDS dataset.

This function writes all non-fixed parameters of the SDDS dataset to the associated binary file. It handles different compression formats such as gzip and LZMA if enabled.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure containing the dataset to write.

Returns

• Returns 1 on successful write of all parameters.
• Returns 0 if an error occurs during the writing process.

The function performs the following steps:

• Validates the dataset structure.
• Iterates over all parameters defined in the dataset layout.
• For each parameter:
  • If it is a fixed value, it is skipped.
  • If the parameter is of type SDDS_STRING, it writes the string using the appropriate compression method.
  • Otherwise, it writes the parameter's value using buffered write functions.

The function handles different file formats:

• For gzip files, it uses SDDS_GZipWriteBinaryString and SDDS_GZipBufferedWrite.
• For LZMA files, it uses SDDS_LZMAWriteBinaryString and SDDS_LZMABufferedWrite.
• For standard binary files, it uses SDDS_WriteBinaryString and SDDS_BufferedWrite.

Note

• The function assumes that the dataset has been properly initialized and that all parameters are correctly allocated.
• Compression support (zLib or LZMA) must be enabled during compilation for handling compressed files.

Definition at line 1426 of file SDDS_binary.c.

                                                               {
  int32_t i;
  SDDS_LAYOUT *layout;
  /*  char *predefined_format; */
  /*  static char buffer[SDDS_MAXLINE]; */
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteBinaryParameters"))
    return (0);
  layout = &SDDS_dataset->layout;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    fBuffer = &SDDS_dataset->fBuffer;
    for (i = 0; i < layout->n_parameters; i++) {
      if (layout->parameter_definition[i].fixed_value)
        continue;
      if (layout->parameter_definition[i].type == SDDS_STRING) {
        if (!SDDS_GZipWriteBinaryString(*((char **)SDDS_dataset->parameter[i]), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteBinaryParameters)");
          return (0);
        }
      } else if (!SDDS_GZipBufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], gzfp, fBuffer)) {
        SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
        return (0);
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_parameters; i++) {
        if (layout->parameter_definition[i].fixed_value)
          continue;
        if (layout->parameter_definition[i].type == SDDS_STRING) {
          if (!SDDS_LZMAWriteBinaryString(*((char **)SDDS_dataset->parameter[i]), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteBinaryParameters)");
            return (0);
          }
        } else if (!SDDS_LZMABufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], lzmafp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
          return (0);
        }
      }
    } else {
      fp = layout->fp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_parameters; i++) {
        if (layout->parameter_definition[i].fixed_value)
          continue;
        if (layout->parameter_definition[i].type == SDDS_STRING) {
          if (!SDDS_WriteBinaryString(*((char **)SDDS_dataset->parameter[i]), fp, fBuffer)) {
            SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteBinaryParameters)");
            return (0);
          }
        } else if (!SDDS_BufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], fp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
          return (0);
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteBinaryRow()

int32_t SDDS_WriteBinaryRow	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	row )

Writes a single binary row of an SDDS dataset to the associated file.

This function writes the data of a specified row within the SDDS dataset to the binary file. It handles different data types, including strings and numeric types, and supports various compression formats such as gzip and LZMA if enabled.

Depending on the dataset's configuration, the function writes directly to a standard binary file, a gzip-compressed file, or an LZMA-compressed file.

Parameters

SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset.
row	The zero-based index of the row to write.

Returns

• Returns 1 on successful writing of the row.
• Returns 0 if an error occurs during the writing process.

The function performs the following steps:

• Validates the dataset structure using SDDS_CheckDataset.
• Determines the file format (standard, gzip, LZMA) and initializes the corresponding file pointer.
• Iterates through each column in the dataset:
  • For string columns, writes the string entry of the specified row using the appropriate write function.
  • For numeric columns, writes the data of the specified row using buffered write functions.
• Handles errors by setting appropriate error messages and aborting the write operation.

Note

• The function assumes that the dataset has been properly initialized and that the specified row exists.
• Compression support (zLib for gzip, LZMA libraries) must be enabled during compilation for handling compressed files.
• The function is not thread-safe and should be called in a synchronized context.

Definition at line 1957 of file SDDS_binary.c.

                                                                     {
  int64_t i, type, size;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteBinaryRow"))
    return (0);
  layout = &SDDS_dataset->layout;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    fBuffer = &SDDS_dataset->fBuffer;
    for (i = 0; i < layout->n_columns; i++) {
      if ((type = layout->column_definition[i].type) == SDDS_STRING) {
        if (!SDDS_GZipWriteBinaryString(*((char **)SDDS_dataset->data[i] + row), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteBinaryRows)");
          return (0);
        }
      } else {
        size = SDDS_type_size[type - 1];
        if (!SDDS_GZipBufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer)) {
          SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteBinaryRow)");
          return (0);
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_columns; i++) {
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!SDDS_LZMAWriteBinaryString(*((char **)SDDS_dataset->data[i] + row), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteBinaryRows)");
            return (0);
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_LZMABufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteBinaryRow)");
            return (0);
          }
        }
      }
    } else {
      fp = layout->fp;
      fBuffer = &SDDS_dataset->fBuffer;
      for (i = 0; i < layout->n_columns; i++) {
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!SDDS_WriteBinaryString(*((char **)SDDS_dataset->data[i] + row), fp, fBuffer)) {
            SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteBinaryRows)");
            return (0);
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_BufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer)) {
            SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteBinaryRow)");
            return (0);
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteBinaryString()

int32_t SDDS_WriteBinaryString	(	char *	string,
		FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer )

Writes a binary string to a file with buffering.

This function writes a binary string to the specified file by first writing the length of the string followed by the string's content to ensure proper binary formatting. If the input string is NULL, an empty string is written instead. The writing operation utilizes a buffered approach to enhance performance.

Parameters

[in]	string	The null-terminated string to be written. If NULL, an empty string is written.
[in]	fp	The file pointer to write to. Must be an open file in binary write mode.
[in,out]	fBuffer	Pointer to the file buffer used for buffered writing operations.

Returns

int32_t Returns 1 on success, 0 on failure.

Return values

1	Operation was successful.
0	An error occurred during writing.

Definition at line 2517 of file SDDS_binary.c.

                                                                                 {
  int32_t length;
  static char *dummy_string = "";
  if (!string)
    string = dummy_string;
  length = strlen(string);
  if (!SDDS_BufferedWrite(&length, sizeof(length), fp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing length");
    return (0);
  }
  if (length && !SDDS_BufferedWrite(string, sizeof(*string) * length, fp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing contents");
    return (0);
  }
  return (1);
}

SDDS_WriteNonNativeBinaryArrays()

int32_t SDDS_WriteNonNativeBinaryArrays

(

SDDS_DATASET *

SDDS_dataset

)

Writes non-native endian binary arrays to an SDDS dataset.

This function iterates through all array definitions in the specified SDDS dataset and writes their binary data to the underlying file. It handles various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. For string arrays, it writes each string individually, ensuring proper memory management and byte order conversion. The function supports different compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files. After writing, it swaps the endianness of the array data to match the system's native byte order.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to write to.

Returns

int32_t Returns 1 on successful writing of all arrays, or 0 if an error occurred.

Return values

1	All non-native endian arrays were successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures, memory allocation issues, or corrupted array definitions.

Note

This function modifies the dataset's array data during the write process. Ensure that the dataset is properly initialized and opened for writing before invoking this function. After writing, the dataset's state is updated to reflect the written arrays.

Definition at line 5350 of file SDDS_binary.c.

                                                                    {
  int32_t i, j, dimension, zero = 0;
  SDDS_LAYOUT *layout;
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
#if defined(zLib)
  gzFile gzfp;
#endif
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteNonNativeBinaryArrays"))
    return (0);
  SDDS_SwapEndsArrayData(SDDS_dataset);
 
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_arrays; i++) {
      if (!SDDS_dataset->array[i].dimension) {
        for (j = 0; j < layout->array_definition[i].dimensions; j++)
          if (!SDDS_GZipBufferedWrite(&zero, sizeof(zero), gzfp, fBuffer)) {
            SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
            SDDS_SwapEndsArrayData(SDDS_dataset);
            return 0;
          }
        continue;
      }
 
      for (j = 0; j < layout->array_definition[i].dimensions; j++) {
        dimension = SDDS_dataset->array[i].dimension[j];
        SDDS_SwapLong(&dimension);
        if (!SDDS_GZipBufferedWrite(&dimension, sizeof(dimension), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
          SDDS_SwapEndsArrayData(SDDS_dataset);
          return (0);
        }
      }
      if (layout->array_definition[i].type == SDDS_STRING) {
        for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
          if (!SDDS_GZipWriteNonNativeBinaryString(((char **)SDDS_dataset->array[i].data)[j], gzfp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryArrays)");
            SDDS_SwapEndsArrayData(SDDS_dataset);
            return (0);
          }
        }
      } else if (!SDDS_GZipBufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, gzfp, fBuffer)) {
        SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteNonNativeBinaryArrays)");
        SDDS_SwapEndsArrayData(SDDS_dataset);
        return (0);
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_arrays; i++) {
        if (!SDDS_dataset->array[i].dimension) {
          for (j = 0; j < layout->array_definition[i].dimensions; j++)
            if (!SDDS_LZMABufferedWrite(&zero, sizeof(zero), lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
              SDDS_SwapEndsArrayData(SDDS_dataset);
              return 0;
            }
          continue;
        }
 
        for (j = 0; j < layout->array_definition[i].dimensions; j++) {
          dimension = SDDS_dataset->array[i].dimension[j];
          SDDS_SwapLong(&dimension);
          if (!SDDS_LZMABufferedWrite(&dimension, sizeof(dimension), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
            SDDS_SwapEndsArrayData(SDDS_dataset);
            return (0);
          }
        }
        if (layout->array_definition[i].type == SDDS_STRING) {
          for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
            if (!SDDS_LZMAWriteNonNativeBinaryString(((char **)SDDS_dataset->array[i].data)[j], lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryArrays)");
              SDDS_SwapEndsArrayData(SDDS_dataset);
              return (0);
            }
          }
        } else if (!SDDS_LZMABufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, lzmafp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteNonNativeBinaryArrays)");
          SDDS_SwapEndsArrayData(SDDS_dataset);
          return (0);
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_arrays; i++) {
        if (!SDDS_dataset->array[i].dimension) {
          for (j = 0; j < layout->array_definition[i].dimensions; j++)
            if (!SDDS_BufferedWrite(&zero, sizeof(zero), fp, fBuffer)) {
              SDDS_SetError("Unable to write null array--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
              SDDS_SwapEndsArrayData(SDDS_dataset);
              return 0;
            }
          continue;
        }
 
        for (j = 0; j < layout->array_definition[i].dimensions; j++) {
          dimension = SDDS_dataset->array[i].dimension[j];
          SDDS_SwapLong(&dimension);
          if (!SDDS_BufferedWrite(&dimension, sizeof(dimension), fp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing dimensions (SDDS_WriteNonNativeBinaryArrays)");
            SDDS_SwapEndsArrayData(SDDS_dataset);
            return (0);
          }
        }
        if (layout->array_definition[i].type == SDDS_STRING) {
          for (j = 0; j < SDDS_dataset->array[i].elements; j++) {
            if (!SDDS_WriteNonNativeBinaryString(((char **)SDDS_dataset->array[i].data)[j], fp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryArrays)");
              SDDS_SwapEndsArrayData(SDDS_dataset);
              return (0);
            }
          }
        } else if (!SDDS_BufferedWrite(SDDS_dataset->array[i].data, SDDS_type_size[layout->array_definition[i].type - 1] * SDDS_dataset->array[i].elements, fp, fBuffer)) {
          SDDS_SetError("Unable to write arrays--failure writing values (SDDS_WriteNonNativeBinaryArrays)");
          SDDS_SwapEndsArrayData(SDDS_dataset);
          return (0);
        }
      }
    }
#if defined(zLib)
  }
#endif
  SDDS_SwapEndsArrayData(SDDS_dataset);
  return (1);
}

SDDS_WriteNonNativeBinaryColumns()

int32_t SDDS_WriteNonNativeBinaryColumns

(

SDDS_DATASET *

SDDS_dataset

)

Writes non-native endian binary columns of an SDDS dataset to the associated file.

This function iterates over each column defined in the SDDS dataset layout and writes its data to the binary file using a non-native byte order. It handles different data types, including strings and numeric types, and supports various compression formats such as gzip and LZMA if enabled.

Depending on the dataset's configuration, the function writes directly to a standard binary file, a gzip-compressed file, or an LZMA-compressed file. It also handles sparse data by only writing rows flagged for inclusion.

Parameters

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to write.

Returns

• Returns 1 on successful writing of all columns.
• Returns 0 if an error occurs during the writing process.

The function performs the following steps:

• Validates the dataset structure using SDDS_CheckDataset.
• Determines the file format (standard, gzip, LZMA) and initializes the corresponding file pointer.
• Iterates through each column in the dataset:
  • For string columns, writes each string entry individually using the appropriate non-native write function.
  • For numeric columns, writes the entire column data in a buffered manner if all rows are flagged; otherwise, writes individual row entries.
• Handles errors by setting appropriate error messages and aborting the write operation.

Note

• The function assumes that the dataset has been properly initialized and populated with data.
• Compression support (zLib for gzip, LZMA libraries) must be enabled during compilation for handling compressed files.
• The function is not thread-safe and should be called in a synchronized context.

Definition at line 1816 of file SDDS_binary.c.

                                                                     {
  int64_t i, row, rows, size, type;
  SDDS_LAYOUT *layout;
#if defined(zLib)
  gzFile gzfp;
#endif
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteNonNativeBinaryColumns"))
    return (0);
  layout = &SDDS_dataset->layout;
  rows = SDDS_CountRowsOfInterest(SDDS_dataset);
  fBuffer = &SDDS_dataset->fBuffer;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_columns; i++) {
      type = layout->column_definition[i].type;
      size = SDDS_type_size[type - 1];
      if (type == SDDS_STRING) {
        for (row = 0; row < SDDS_dataset->n_rows; row++) {
          if (SDDS_dataset->row_flag[row] && !SDDS_GZipWriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), gzfp, fBuffer)) {
            SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryColumns)");
            return (0);
          }
        }
      } else {
        if (rows == SDDS_dataset->n_rows) {
          if (!SDDS_GZipBufferedWrite((char *)SDDS_dataset->data[i], size * rows, gzfp, fBuffer)) {
            SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
            return (0);
          }
        } else {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_GZipBufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
              return (0);
            }
          }
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_columns; i++) {
        type = layout->column_definition[i].type;
        size = SDDS_type_size[type - 1];
        if (type == SDDS_STRING) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_LZMAWriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryColumns)");
              return (0);
            }
          }
        } else {
          if (rows == SDDS_dataset->n_rows) {
            if (!SDDS_LZMABufferedWrite(SDDS_dataset->data[i], size * rows, lzmafp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
              return (0);
            }
          } else {
            for (row = 0; row < SDDS_dataset->n_rows; row++) {
              if (SDDS_dataset->row_flag[row] && !SDDS_LZMABufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer)) {
                SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
                return (0);
              }
            }
          }
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_columns; i++) {
        type = layout->column_definition[i].type;
        size = SDDS_type_size[type - 1];
        if (type == SDDS_STRING) {
          for (row = 0; row < SDDS_dataset->n_rows; row++) {
            if (SDDS_dataset->row_flag[row] && !SDDS_WriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), fp, fBuffer)) {
              SDDS_SetError("Unable to write arrays--failure writing string (SDDS_WriteNonNativeBinaryColumns)");
              return (0);
            }
          }
        } else {
          if (rows == SDDS_dataset->n_rows) {
            if (!SDDS_BufferedWrite(SDDS_dataset->data[i], size * rows, fp, fBuffer)) {
              SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
              return (0);
            }
          } else {
            for (row = 0; row < SDDS_dataset->n_rows; row++) {
              if (SDDS_dataset->row_flag[row] && !SDDS_BufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer)) {
                SDDS_SetError("Unable to write columns--failure writing values (SDDS_WriteNonNativeBinaryColumns)");
                return (0);
              }
            }
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteNonNativeBinaryPage()

int32_t SDDS_WriteNonNativeBinaryPage

(

SDDS_DATASET *

SDDS_dataset

)

Writes a non-native endian binary page to an SDDS dataset.

This function writes a binary page to the specified SDDS dataset, handling byte order reversal to convert between little-endian and big-endian formats. It manages various compression formats, including uncompressed, GZIP-compressed, and LZMA-compressed files. The function performs the following operations:

• Counts the number of rows to write.
• Writes the row count with appropriate byte order handling.
• Writes non-native endian parameters and arrays.
• Writes column data in either column-major or row-major format based on the dataset's configuration.
• Flushes the buffer to ensure all data is written to the file.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to write to.

Returns

int32_t Returns 1 on successful writing of the binary page, or 0 if an error occurred.

Return values

1	The binary page was successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures, memory allocation issues, or corrupted dataset definitions.

Note

This function modifies the dataset's internal structures during the write process. Ensure that the dataset is properly initialized and opened for writing before invoking this function. After writing, the dataset's state is updated to reflect the newly written page.

Definition at line 4986 of file SDDS_binary.c.

{
  FILE *fp;
  struct lzmafile *lzmafp = NULL;
  int64_t i, rows, fixed_rows;
  int32_t min32 = INT32_MIN, rows32;
  SDDS_FILEBUFFER *fBuffer;
#if defined(zLib)
  gzFile gzfp = NULL;
#endif
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteNonNativeBinaryPage"))
    return (0);
  if (!(fp = SDDS_dataset->layout.fp)) {
    SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteNonNativeBinaryPage)");
    return (0);
  }
  fBuffer = &SDDS_dataset->fBuffer;
 
  if (!fBuffer->buffer) {
    if (!(fBuffer->buffer = fBuffer->data = SDDS_Malloc(sizeof(char) * defaultIOBufferSize))) {
      SDDS_SetError("Unable to do buffered read--allocation failure (SDDS_WriteNonNativeBinaryPage)");
      return 0;
    }
    fBuffer->bufferSize = defaultIOBufferSize;
    fBuffer->bytesLeft = defaultIOBufferSize;
  }
  SDDS_SwapLong(&min32);
 
  rows = SDDS_CountRowsOfInterest(SDDS_dataset);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!(gzfp = SDDS_dataset->layout.gzfp)) {
      SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteNonNativeBinaryPage)");
      return (0);
    }
    SDDS_dataset->rowcount_offset = gztell(gzfp);
    if (SDDS_dataset->layout.data_mode.fixed_row_count) {
      fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
      if (fixed_rows > INT32_MAX) {
        if (!SDDS_GZipBufferedWrite(&min32, sizeof(min32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&fixed_rows);
        if (!SDDS_GZipBufferedWrite(&fixed_rows, sizeof(fixed_rows), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&fixed_rows);
      } else {
        rows32 = (int32_t)fixed_rows;
        SDDS_SwapLong(&rows32);
        if (!SDDS_GZipBufferedWrite(&rows32, sizeof(rows32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
      }
    } else {
      if (rows > INT32_MAX) {
        if (!SDDS_GZipBufferedWrite(&min32, sizeof(min32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&rows);
        if (!SDDS_GZipBufferedWrite(&rows, sizeof(rows), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
        SDDS_SwapLong64(&rows);
      } else {
        rows32 = (int32_t)rows;
        SDDS_SwapLong(&rows32);
        if (!SDDS_GZipBufferedWrite(&rows32, sizeof(rows32), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
          return (0);
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!(lzmafp = SDDS_dataset->layout.lzmafp)) {
        SDDS_SetError("Unable to write page--file pointer is NULL (SDDS_WriteNonNativeBinaryPage)");
        return (0);
      }
      SDDS_dataset->rowcount_offset = lzma_tell(lzmafp);
      if (SDDS_dataset->layout.data_mode.fixed_row_count) {
        fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
        if (fixed_rows > INT32_MAX) {
          if (!SDDS_LZMABufferedWrite(&min32, sizeof(min32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&fixed_rows);
          if (!SDDS_LZMABufferedWrite(&fixed_rows, sizeof(fixed_rows), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&fixed_rows);
        } else {
          rows32 = (int32_t)fixed_rows;
          SDDS_SwapLong(&rows32);
          if (!SDDS_LZMABufferedWrite(&rows32, sizeof(rows32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
        }
      } else {
        if (rows > INT32_MAX) {
          if (!SDDS_LZMABufferedWrite(&min32, sizeof(min32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&rows);
          if (!SDDS_LZMABufferedWrite(&rows, sizeof(rows), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&rows);
        } else {
          rows32 = (int32_t)rows;
          SDDS_SwapLong(&rows32);
          if (!SDDS_LZMABufferedWrite(&rows32, sizeof(rows32), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
        }
      }
    } else {
      SDDS_dataset->rowcount_offset = ftell(fp);
      if (SDDS_dataset->layout.data_mode.fixed_row_count) {
        fixed_rows = ((rows / SDDS_dataset->layout.data_mode.fixed_row_increment) + 2) * SDDS_dataset->layout.data_mode.fixed_row_increment;
        if (fixed_rows > INT32_MAX) {
          if (!SDDS_BufferedWrite(&min32, sizeof(min32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&fixed_rows);
          if (!SDDS_BufferedWrite(&fixed_rows, sizeof(fixed_rows), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&fixed_rows);
        } else {
          rows32 = (int32_t)fixed_rows;
          SDDS_SwapLong(&rows32);
          if (!SDDS_BufferedWrite(&rows32, sizeof(rows32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
        }
      } else {
        if (rows > INT32_MAX) {
          if (!SDDS_BufferedWrite(&min32, sizeof(min32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&rows);
          if (!SDDS_BufferedWrite(&rows, sizeof(rows), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
          SDDS_SwapLong64(&rows);
        } else {
          rows32 = (int32_t)rows;
          SDDS_SwapLong(&rows32);
          if (!SDDS_BufferedWrite(&rows32, sizeof(rows32), fp, fBuffer)) {
            SDDS_SetError("Unable to write page--failure writing number of rows (SDDS_WriteNonNativeBinaryPage)");
            return (0);
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  if (!SDDS_WriteNonNativeBinaryParameters(SDDS_dataset)) {
    SDDS_SetError("Unable to write page--parameter writing problem (SDDS_WriteNonNativeBinaryPage)");
    return 0;
  }
  if (!SDDS_WriteNonNativeBinaryArrays(SDDS_dataset)) {
    SDDS_SetError("Unable to write page--array writing problem (SDDS_WriteNonNativeBinaryPage)");
    return 0;
  }
  SDDS_SwapEndsColumnData(SDDS_dataset);
  if (SDDS_dataset->layout.n_columns) {
    if (SDDS_dataset->layout.data_mode.column_major) {
      if (!SDDS_WriteNonNativeBinaryColumns(SDDS_dataset)) {
        SDDS_SetError("Unable to write page--column writing problem (SDDS_WriteNonNativeBinaryPage)");
        return 0;
      }
    } else {
      for (i = 0; i < SDDS_dataset->n_rows; i++) {
        if (SDDS_dataset->row_flag[i]) {
          if (!SDDS_WriteNonNativeBinaryRow(SDDS_dataset, i)) {
            SDDS_SetError("Unable to write page--row writing problem (SDDS_WriteNonNativeBinaryPage)");
            return 0;
          }
        }
      }
    }
  }
  SDDS_SwapEndsColumnData(SDDS_dataset);
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!SDDS_GZipFlushBuffer(gzfp, fBuffer)) {
      SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteNonNativeBinaryPage)");
      return 0;
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!SDDS_LZMAFlushBuffer(lzmafp, fBuffer)) {
        SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteNonNativeBinaryPage)");
        return 0;
      }
    } else {
      if (!SDDS_FlushBuffer(fp, fBuffer)) {
        SDDS_SetError("Unable to write page--buffer flushing problem (SDDS_WriteNonNativeBinaryPage)");
        return 0;
      }
    }
#if defined(zLib)
  }
#endif
  SDDS_dataset->last_row_written = SDDS_dataset->n_rows - 1;
  SDDS_dataset->n_rows_written = rows;
  SDDS_dataset->writing_page = 1;
  return (1);
}

SDDS_WriteNonNativeBinaryParameters()

int32_t SDDS_WriteNonNativeBinaryParameters

(

SDDS_DATASET *

SDDS_dataset

)

Writes non-native endian binary parameters to an SDDS dataset.

This function iterates through all parameter definitions in the specified SDDS dataset and writes their binary data to the underlying file. It handles various data types, including short, unsigned short, long, unsigned long, long long, unsigned long long, float, double, and long double. For string parameters, it writes each string individually, ensuring proper memory management and byte order conversion. Parameters with fixed values are skipped during the write process. The function supports different compression formats, including uncompressed, LZMA-compressed, and GZIP-compressed files.

Parameters

[in,out]

SDDS_dataset

Pointer to the SDDS_DATASET structure representing the dataset to write to.

Returns

int32_t Returns 1 on successful writing of all parameters, or 0 if an error occurred.

Return values

1	All non-native endian parameters were successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures, memory allocation issues, or corrupted parameter definitions.

Note

This function modifies the dataset's parameter data during the write process. Ensure that the dataset is properly initialized and opened for writing before invoking this function. After writing, the dataset's state is updated to reflect the written parameters.

Definition at line 5242 of file SDDS_binary.c.

                                                                        {
  int32_t i;
  SDDS_LAYOUT *layout;
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
#if defined(zLib)
  gzFile gzfp;
#endif
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteNonNativeBinaryParameters"))
    return (0);
 
  SDDS_SwapEndsParameterData(SDDS_dataset);
 
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    if (!(gzfp = layout->gzfp)) {
      SDDS_SetError("Unable to write parameters--file pointer is NULL (SDDS_WriteNonNativeBinaryParameters)");
      return (0);
    }
    for (i = 0; i < layout->n_parameters; i++) {
      if (layout->parameter_definition[i].fixed_value)
        continue;
      if (layout->parameter_definition[i].type == SDDS_STRING) {
        if (!SDDS_GZipWriteNonNativeBinaryString(*((char **)SDDS_dataset->parameter[i]), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteNonNativeBinaryParameters)");
          SDDS_SwapEndsParameterData(SDDS_dataset);
          return (0);
        }
      } else if (!SDDS_GZipBufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], gzfp, fBuffer)) {
        SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
        SDDS_SwapEndsParameterData(SDDS_dataset);
        return (0);
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      if (!(lzmafp = layout->lzmafp)) {
        SDDS_SetError("Unable to write parameters--file pointer is NULL (SDDS_WriteNonNativeBinaryParameters)");
        return (0);
      }
      for (i = 0; i < layout->n_parameters; i++) {
        if (layout->parameter_definition[i].fixed_value)
          continue;
        if (layout->parameter_definition[i].type == SDDS_STRING) {
          if (!SDDS_LZMAWriteNonNativeBinaryString(*((char **)SDDS_dataset->parameter[i]), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteNonNativeBinaryParameters)");
            SDDS_SwapEndsParameterData(SDDS_dataset);
            return (0);
          }
        } else if (!SDDS_LZMABufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], lzmafp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
          SDDS_SwapEndsParameterData(SDDS_dataset);
          return (0);
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_parameters; i++) {
        if (layout->parameter_definition[i].fixed_value)
          continue;
        if (layout->parameter_definition[i].type == SDDS_STRING) {
          if (!SDDS_WriteNonNativeBinaryString(*((char **)SDDS_dataset->parameter[i]), fp, fBuffer)) {
            SDDS_SetError("Unable to write parameters--failure writing string (SDDS_WriteNonNativeBinaryParameters)");
            SDDS_SwapEndsParameterData(SDDS_dataset);
            return (0);
          }
        } else if (!SDDS_BufferedWrite(SDDS_dataset->parameter[i], SDDS_type_size[layout->parameter_definition[i].type - 1], fp, fBuffer)) {
          SDDS_SetError("Unable to write parameters--failure writing value (SDDS_WriteBinaryParameters)");
          SDDS_SwapEndsParameterData(SDDS_dataset);
          return (0);
        }
      }
    }
#if defined(zLib)
  }
#endif
 
  SDDS_SwapEndsParameterData(SDDS_dataset);
  return (1);
}

SDDS_WriteNonNativeBinaryRow()

int32_t SDDS_WriteNonNativeBinaryRow	(	SDDS_DATASET *	SDDS_dataset,
		int64_t	row )

Writes a non-native endian binary row to an SDDS dataset.

This function writes a single row of data to the specified SDDS dataset, handling byte order reversal to convert between little-endian and big-endian formats. It supports various compression formats, including uncompressed, GZIP-compressed, and LZMA-compressed files. The function iterates through all column definitions, writing each column's data appropriately based on its type. For string columns, it ensures proper memory management and byte order conversion by utilizing specialized string writing functions. For non-string data types, it writes the binary data directly with the correct byte ordering.

Parameters

[in,out]	SDDS_dataset	Pointer to the SDDS_DATASET structure representing the dataset to write to.
[in]	row	The index of the row to write to the dataset.

Returns

int32_t Returns 1 on successful writing of the binary row, or 0 if an error occurred.

Return values

1	The binary row was successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures or corrupted data.

Note

This function modifies the dataset's internal data structures during the write process. Ensure that the dataset is properly initialized and opened for writing before invoking this function. After writing, the dataset's state is updated to reflect the newly written row.

Definition at line 5505 of file SDDS_binary.c.

                                                                              {
  int64_t i, type, size;
  SDDS_LAYOUT *layout;
  FILE *fp;
  struct lzmafile *lzmafp;
  SDDS_FILEBUFFER *fBuffer;
#if defined(zLib)
  gzFile gzfp;
#endif
 
  if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_WriteNonNativeBinaryRow"))
    return (0);
  layout = &SDDS_dataset->layout;
  fBuffer = &SDDS_dataset->fBuffer;
#if defined(zLib)
  if (SDDS_dataset->layout.gzipFile) {
    gzfp = layout->gzfp;
    for (i = 0; i < layout->n_columns; i++) {
      if ((type = layout->column_definition[i].type) == SDDS_STRING) {
        if (!SDDS_GZipWriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), gzfp, fBuffer)) {
          SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteNonNativeBinaryRows)");
          return (0);
        }
      } else {
        size = SDDS_type_size[type - 1];
        if (!SDDS_GZipBufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, gzfp, fBuffer)) {
          SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteNonNativeBinaryRow)");
          return (0);
        }
      }
    }
  } else {
#endif
    if (SDDS_dataset->layout.lzmaFile) {
      lzmafp = layout->lzmafp;
      for (i = 0; i < layout->n_columns; i++) {
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!SDDS_LZMAWriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteNonNativeBinaryRows)");
            return (0);
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_LZMABufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, lzmafp, fBuffer)) {
            SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteNonNativeBinaryRow)");
            return (0);
          }
        }
      }
    } else {
      fp = layout->fp;
      for (i = 0; i < layout->n_columns; i++) {
        if ((type = layout->column_definition[i].type) == SDDS_STRING) {
          if (!SDDS_WriteNonNativeBinaryString(*((char **)SDDS_dataset->data[i] + row), fp, fBuffer)) {
            SDDS_SetError("Unable to write rows--failure writing string (SDDS_WriteNonNativeBinaryRows)");
            return (0);
          }
        } else {
          size = SDDS_type_size[type - 1];
          if (!SDDS_BufferedWrite((char *)SDDS_dataset->data[i] + row * size, size, fp, fBuffer)) {
            SDDS_SetError("Unable to write row--failure writing value (SDDS_WriteNonNativeBinaryRow)");
            return (0);
          }
        }
      }
    }
#if defined(zLib)
  }
#endif
  return (1);
}

SDDS_WriteNonNativeBinaryString()

int32_t SDDS_WriteNonNativeBinaryString	(	char *	string,
		FILE *	fp,
		SDDS_FILEBUFFER *	fBuffer )

Writes a non-native endian binary string to a file.

This function writes a binary string to the specified file pointer, handling non-native endianness. It first writes the length of the string as a 32-bit integer with byte order swapped. If the string is not to be skipped, it then writes the string data itself followed by a null terminator. If the input string is NULL, an empty string is written instead.

Parameters

[in]	string	The string to write. If NULL, an empty string is written.
[in]	fp	Pointer to the FILE where the string will be written.
[in]	fBuffer	Pointer to the SDDS_FILEBUFFER structure used for buffered writing.

Returns

int32_t Returns 1 on successful writing of the string, or 0 if an error occurred.

Return values

1	The string was successfully written and byte-swapped.
0	An error occurred during the write operation, such as I/O failures or memory allocation issues.

Note

The caller is responsible for ensuring that the file pointer fp is valid and open for writing. This function does not perform memory allocation for the string; it assumes that the string is already allocated and managed appropriately.

Definition at line 5597 of file SDDS_binary.c.

                                                                                          {
  int32_t length;
  static char *dummy_string = "";
  if (!string)
    string = dummy_string;
  length = strlen(string);
  SDDS_SwapLong(&length);
  if (!SDDS_BufferedWrite(&length, sizeof(length), fp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing length");
    return (0);
  }
  SDDS_SwapLong(&length);
  if (length && !SDDS_BufferedWrite(string, sizeof(*string) * length, fp, fBuffer)) {
    SDDS_SetError("Unable to write string--error writing contents");
    return (0);
  }
  return (1);
}