sddsnormalize.c File Reference

Detailed Description

A program for SDDS-format column normalization.

This program normalizes the specified columns of an SDDS file based on user-defined options. It provides various statistical modes for normalization, supports multithreading, and offers flexibility in data handling with features like custom suffixes and column exclusions.

Usage

sddsnormalize [<inputfile>] [<outputfile>]
              [-pipe=[input][,output]]
              -columns=[mode=<mode>,][suffix=<string>,][exclude=<wildcardString>,][functionOf=<columnName>,]<columnName>[,...]
              [-threads=<number>]
              [-majorOrder=row|column]

Options

Required	Description
-columns	Specifies the columns to normalize and their modes of normalization.

Optional	Description
-pipe	Specifies whether the input/output is piped.
-threads	Defines the number of threads for parallel normalization.
-majorOrder	Sets the processing order as row-major or column-major.

Column Modes	Description
minimum	Use the minimum value as the normalization factor.
maximum	Use the maximum value as the normalization factor.
largest	Use the larger of |min| or |max| (default).
signedlargest	Use the largest value with its sign retained.
spread	Use (max - min) as the normalization factor.
rms	Use the root-mean-square of the values.
standarddeviation	Use the n-1 weighted standard deviation.
sum	Use the sum of all values.
area	Use the area under the curve (requires functionOf).
average	Use the average of all values.

Specific Requirements

• For the area mode in -columns, the functionOf qualifier must be provided.
• The -columns mode defaults to largest if not specified.

Copyright

• (c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
• (c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.

License

This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.

Author

M. Borland, R. Soliday, H. Shang

Definition in file sddsnormalize.c.

#include "mdb.h"
#include "SDDS.h"
#include "scan.h"
#include <ctype.h>

Go to the source code of this file.

Functions
long	resolveColumnNames (SDDS_DATASET *SDDSin, NORM_REQUEST *normRequest, long normRequests, NORM_SPEC **normSpecRet, long *normSpecsRet)
int	main (int argc, char **argv)

Function Documentation

main()

int main	(	int	argc,
		char **	argv )

Definition at line 156 of file sddsnormalize.c.

                                {
  int iArg;
  NORM_REQUEST *normRequest;
  NORM_SPEC *normSpec;
  long normRequests, normSpecs, i, readCode;
  int64_t j, rows;
  char *input, *output, *modeString;
  unsigned long pipeFlags, majorOrderFlag;
  SCANNED_ARG *scanned;
  SDDS_DATASET SDDSin, SDDSout;
  double *data, *funcOfData, factor, min, max;
  short columnMajorOrder = -1;
  int threads = 1;
 
  SDDS_RegisterProgramName(argv[0]);
  argc = scanargs(&scanned, argc, argv);
  if (argc < 3)
    bomb(NULL, USAGE);
 
  output = input = NULL;
  pipeFlags = 0;
  normRequest = NULL;
  normSpec = NULL;
  normRequests = normSpecs = 0;
 
  for (iArg = 1; iArg < argc; iArg++) {
    if (scanned[iArg].arg_type == OPTION) {
      /* process options here */
      switch (match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
      case CLO_MAJOR_ORDER:
        majorOrderFlag = 0;
        scanned[iArg].n_items--;
        if (scanned[iArg].n_items > 0 && (!scanItemList(&majorOrderFlag, scanned[iArg].list + 1, &scanned[iArg].n_items, 0, "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER, "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL)))
          SDDS_Bomb("invalid -majorOrder syntax/values");
        if (majorOrderFlag & SDDS_COLUMN_MAJOR_ORDER)
          columnMajorOrder = 1;
        else if (majorOrderFlag & SDDS_ROW_MAJOR_ORDER)
          columnMajorOrder = 0;
        break;
      case CLO_COLUMNS:
        if (!(normRequest = SDDS_Realloc(normRequest, sizeof(*normRequest) * (normRequests + 1))))
          SDDS_Bomb("memory allocation failure");
        normRequest[normRequests].exclude = normRequest[normRequests].suffix = NULL;
        if (!scanItemList(&normRequest[normRequests].flags,
                          scanned[iArg].list, &scanned[iArg].n_items,
                          SCANITEMLIST_UNKNOWN_VALUE_OK | SCANITEMLIST_REMOVE_USED_ITEMS |
                          SCANITEMLIST_IGNORE_VALUELESS,
                          "mode", SDDS_STRING, &modeString, 1, FL_MODE_GIVEN,
                          "suffix", SDDS_STRING, &normRequest[normRequests].suffix, 1, FL_SUFFIX_GIVEN,
                          "functionof", SDDS_STRING, &normRequest[normRequests].functionOf, 1, FL_FUNCOF_GIVEN,
                          "exclude", SDDS_STRING, &normRequest[normRequests].exclude, 1, 0, NULL))
          SDDS_Bomb("invalid -columns syntax");
        if (normRequest[normRequests].flags & FL_MODE_GIVEN) {
          if ((normRequest[normRequests].mode = match_string(modeString, normMode, NORM_OPTIONS, 0)) < 0)
            SDDS_Bomb("invalid -columns syntax: unknown mode");
        } else
          normRequest[normRequests].mode = NORM_LARGEST;
        if (scanned[iArg].n_items < 1)
          SDDS_Bomb("invalid -columns syntax: no columns listed");
        normRequest[normRequests].source = scanned[iArg].list + 1;
        normRequest[normRequests].sources = scanned[iArg].n_items - 1;
        normRequests++;
        break;
      case CLO_THREADS:
        if (scanned[iArg].n_items != 2 ||
            !sscanf(scanned[iArg].list[1], "%d", &threads) || threads < 1)
          SDDS_Bomb("invalid -threads syntax");
        break;
      case CLO_PIPE:
        if (!processPipeOption(scanned[iArg].list + 1, scanned[iArg].n_items - 1, &pipeFlags))
          SDDS_Bomb("invalid -pipe syntax");
        break;
      default:
        fprintf(stderr, "error: unknown/ambiguous option: %s\n", scanned[iArg].list[0]);
        exit(EXIT_FAILURE);
        break;
      }
    } else {
      if (!input)
        input = scanned[iArg].list[0];
      else if (!output)
        output = scanned[iArg].list[0];
      else
        SDDS_Bomb("too many filenames seen");
    }
  }
 
  processFilenames("sddsnormalize", &input, &output, pipeFlags, 0, NULL);
 
  if (!normRequests)
    SDDS_Bomb("supply the names of columns to normalize with the -columns option");
 
  if (!SDDS_InitializeInput(&SDDSin, input))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
 
  if (!resolveColumnNames(&SDDSin, normRequest, normRequests, &normSpec, &normSpecs))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
 
  if (!normSpecs)
    SDDS_Bomb("no columns selected for normalization");
 
  if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
  if (columnMajorOrder != -1)
    SDDSout.layout.data_mode.column_major = columnMajorOrder;
  else
    SDDSout.layout.data_mode.column_major = SDDSin.layout.data_mode.column_major;
  for (i = 0; i < normSpecs; i++) {
    if (normSpec[i].flags & FL_SUFFIX_GIVEN) {
      if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, normSpec[i].source, normSpec[i].target) ||
          !SDDS_ChangeColumnInformation(&SDDSout, "units", "", SDDS_BY_NAME, normSpec[i].target))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    } else if (!SDDS_ChangeColumnInformation(&SDDSout, "units", "Normalized", SDDS_BY_NAME, normSpec[i].target))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
  }
 
  if (!SDDS_WriteLayout(&SDDSout))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
 
  while ((readCode = SDDS_ReadPage(&SDDSin)) > 0) {
    if (!SDDS_CopyPage(&SDDSout, &SDDSin))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    if ((rows = SDDS_RowCount(&SDDSin))) {
      for (i = 0; i < normSpecs; i++) {
        if (!(data = SDDS_GetColumnInDoubles(&SDDSin, normSpec[i].source)))
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        funcOfData = NULL;
        if (normSpec[i].functionOf &&
            !(funcOfData = SDDS_GetColumnInDoubles(&SDDSin, normSpec[i].functionOf)))
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        if (!find_min_max(&min, &max, data, rows))
          min = max = 1;
        switch (normSpec[i].mode) {
        case NORM_RMS:
          factor = rmsValueThreaded(data, rows, threads);
          break;
        case NORM_STDEV:
          factor = standardDeviationThreaded(data, rows, threads);
          break;
        case NORM_MINIMUM:
          factor = min;
          break;
        case NORM_MAXIMUM:
          factor = max;
          break;
        case NORM_LARGEST:
          min = fabs(min);
          max = fabs(max);
          factor = MAX(min, max);
          break;
        case NORM_SLARGEST:
          if (fabs(min) > fabs(max))
            factor = min;
          else
            factor = max;
          break;
        case NORM_SPREAD:
          factor = max - min;
          break;
        case NORM_SUM:
          for (j = factor = 0; j < rows; j++)
            factor += data[j];
          break;
        case NORM_AREA:
          if (!funcOfData)
            SDDS_Bomb("functionOf qualifier must be given for area normalization");
          trapazoidIntegration(funcOfData, data, rows, &factor);
          break;
        case NORM_AVERAGE:
          for (j = factor = 0; j < rows; j++)
            factor += data[j];
          factor /= rows;
          break;
        default:
          SDDS_Bomb("Invalid normalization mode---programming error");
          break;
        }
        if (funcOfData)
          free(funcOfData);
        if (factor)
          for (j = 0; j < rows; j++)
            data[j] /= factor;
        if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME, data, rows, normSpec[i].target))
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        free(data);
      }
    }
    if (!SDDS_WritePage(&SDDSout))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
  }
  if (!SDDS_Terminate(&SDDSin)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    return EXIT_FAILURE;
  }
  if (!SDDS_Terminate(&SDDSout)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    return EXIT_FAILURE;
  }
 
  return EXIT_SUCCESS;
}

resolveColumnNames()

long resolveColumnNames	(	SDDS_DATASET *	SDDSin,
		NORM_REQUEST *	normRequest,
		long	normRequests,
		NORM_SPEC **	normSpecRet,
		long *	normSpecsRet )

Definition at line 358 of file sddsnormalize.c.

                                                                                                                                         {
  long i, j;
  int32_t columns;
  char **column, buffer[1024];
  long normSpecs = 0;
  NORM_SPEC *normSpec = NULL;
 
  for (i = 0; i < normRequests; i++) {
    SDDS_SetColumnFlags(SDDSin, 0);
    if (normRequest[i].flags & FL_SUFFIX_GIVEN) {
      if (!normRequest[i].suffix || !strlen(normRequest[i].suffix)) {
        SDDS_SetError("resolveColumnNames: missing or blank suffix");
        return 0;
      }
    }
    for (j = 0; j < normRequest[i].sources; j++) {
      if (!SDDS_SetColumnsOfInterest(SDDSin, SDDS_MATCH_STRING, normRequest[i].source[j], SDDS_OR)) {
        SDDS_SetError("resolveColumnNames: SDDS_SetColumnsOfInterest error");
        return 0;
      }
    }
    if (normRequest[i].exclude &&
        !SDDS_SetColumnsOfInterest(SDDSin, SDDS_MATCH_STRING, normRequest[i].exclude, SDDS_NEGATE_MATCH | SDDS_AND)) {
      SDDS_SetError("resolveColumnNames: SDDS_SetColumnsOfInterest error");
      return 0;
    }
    if (!(column = SDDS_GetColumnNames(SDDSin, &columns)) || columns == 0) {
      sprintf(buffer, "No match for column list: ");
      for (j = 0; j < normRequest[i].sources; j++) {
        strcat(buffer, normRequest[i].source[j]);
        if (j != normRequest[i].sources - 1)
          strcat(buffer, ", ");
      }
      SDDS_SetError(buffer);
      return 0;
    }
    if (!(normSpec = SDDS_Realloc(normSpec, sizeof(*normSpec) * (normSpecs + columns)))) {
      SDDS_SetError("resolveColumnNames: Memory allocation failure");
      return 0;
    }
    for (j = 0; j < columns; j++) {
      normSpec[j + normSpecs].source = column[j];
      normSpec[j + normSpecs].mode = normRequest[i].mode;
      normSpec[j + normSpecs].flags = normRequest[i].flags;
      normSpec[j + normSpecs].functionOf = NULL;
      if (normRequest[i].flags & FL_FUNCOF_GIVEN) {
        if (!SDDS_CopyString(&normSpec[j + normSpecs].functionOf, normRequest[i].functionOf)) {
          SDDS_SetError("resolveColumnNames: Memory allocation failure");
          return 0;
        }
      }
      normSpec[j + normSpecs].target = NULL;
      if (normRequest[i].flags & FL_SUFFIX_GIVEN) {
        sprintf(buffer, "%s%s", normSpec[j + normSpecs].source, normRequest[i].suffix);
        if (!SDDS_CopyString(&normSpec[j + normSpecs].target, buffer)) {
          SDDS_SetError("resolveColumnNames: Memory allocation failure");
          return 0;
        }
      } else
        normSpec[j + normSpecs].target = normSpec[j + normSpecs].source;
    }
    normSpecs += columns;
  }
  *normSpecRet = normSpec;
  *normSpecsRet = normSpecs;
  return 1;
}