A utility program for creating SDDS data sets based on matching data. More...

#include "mdb.h"
#include "SDDS.h"
#include "scan.h"

Typedefs
typedef char *	STRING_PAIR[2]

Enumerations
enum	option_type { SET_MATCH_COLUMNS , SET_EQUATE_COLUMNS , SET_NOWARNINGS , SET_INVERT , SET_REUSE , SET_PIPE , SET_MAJOR_ORDER , N_OPTIONS }

Functions
long	rows_equate (SDDS_DATASET SDDS1, int64_t row1, SDDS_DATASET SDDS2, int64_t row2, long equate_columns, STRING_PAIR *equate_column)

int	main (int argc, char **argv)

Variables
char *	option [N_OPTIONS]

char *	USAGE

Detailed Description

A utility program for creating SDDS data sets based on matching data.

This program creates an SDDS (Self Describing Data Set) file from an input dataset by selecting rows based on matching or equating data from a second dataset. It provides flexibility for data matching, inversion, and reuse of rows, and can handle both row-major and column-major data ordering.

Features:

Match rows in the input dataset against another dataset using specified columns.
Equate numerical columns for advanced row selection.
Options to invert selection logic and reuse rows from the second dataset.
Output can be formatted in row-major or column-major order.

Usage:

sddsmselect [OPTIONS] [<input1>] <input2> [<output>]

Options:

-pipe[=input][,output]: Use pipe for input and/or output.
-match=<col1>=<col2>,...: Specify columns to match between datasets.
-equate=<col1>=<col2>,...: Specify columns to equate between datasets.
-invert: Select rows with no matching rows in the second dataset.
-reuse[=rows|page]: Allow reuse of rows from the second dataset.
-majorOrder=row|column: Set the output file order.
-nowarnings: Suppress warning messages.

Example:

sddsmselect -match=ID=ID1 input1.sdds input2.sdds output.sdds

Copyright

(c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
(c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.

License: This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.

Author: M. Borland, C. Saunders, R. Soliday, H. Shang

Definition in file sddsmselect.c.

Typedef Documentation

◆ STRING_PAIR

typedef char* STRING_PAIR[2]

Definition at line 62 of file sddsmselect.c.

Enumeration Type Documentation

◆ option_type

enum option_type

Definition at line 51 of file sddsmselect.c.

                 {
  SET_MATCH_COLUMNS,
  SET_EQUATE_COLUMNS,
  SET_NOWARNINGS,
  SET_INVERT,
  SET_REUSE,
  SET_PIPE,
  SET_MAJOR_ORDER,
  N_OPTIONS
};

Function Documentation

◆ main()

int main	(	int	argc,
		char **	argv )

Definition at line 107 of file sddsmselect.c.

                                {
  SDDS_DATASET SDDS_1, SDDS_2, SDDS_output;
  long i, i_arg, reuse, reusePage;
  int64_t j, k, rows1, rows2, n, outputRow;
  SCANNED_ARG *s_arg;
  char s[200], *ptr;
  STRING_PAIR *match_column, *equate_column;
  long match_columns, equate_columns;
  char *input1, *input2, *output, *match_value;
  long tmpfile_used, retval1, retval2;
  long *row_used, warnings, invert;
  unsigned long pipeFlags, majorOrderFlag;
  short columnMajorOrder = -1;
 
  SDDS_RegisterProgramName(argv[0]);
  argc = scanargs(&s_arg, argc, argv);
  if (argc < 3)
    bomb(NULL, USAGE);
 
  input1 = input2 = output = NULL;
  match_column = equate_column = NULL;
  match_columns = equate_columns = reuse = reusePage = invert = 0;
  tmpfile_used = 0;
  warnings = 1;
  pipeFlags = 0;
 
  for (i_arg = 1; i_arg < argc; i_arg++) {
    if (s_arg[i_arg].arg_type == OPTION) {
      delete_chars(s_arg[i_arg].list[0], "_");
      switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
      case SET_MAJOR_ORDER:
        majorOrderFlag = 0;
        s_arg[i_arg].n_items--;
        if (s_arg[i_arg].n_items > 0 &&
            (!scanItemList(&majorOrderFlag, s_arg[i_arg].list + 1, &s_arg[i_arg].n_items,
                           0, "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
                           "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL)))
          SDDS_Bomb("Invalid -majorOrder syntax or values.");
        if (majorOrderFlag & SDDS_COLUMN_MAJOR_ORDER)
          columnMajorOrder = 1;
        else if (majorOrderFlag & SDDS_ROW_MAJOR_ORDER)
          columnMajorOrder = 0;
        break;
      case SET_MATCH_COLUMNS:
        if (s_arg[i_arg].n_items < 2)
          SDDS_Bomb("Invalid -match syntax.");
        match_column = trealloc(match_column, sizeof(*match_column) * (match_columns + s_arg[i_arg].n_items - 1));
        for (i = 1; i < s_arg[i_arg].n_items; i++) {
          if ((ptr = strchr(s_arg[i_arg].list[i], '=')))
            *ptr++ = 0;
          else
            ptr = s_arg[i_arg].list[i];
          match_column[i - 1 + match_columns][0] = s_arg[i_arg].list[i];
          match_column[i - 1 + match_columns][1] = ptr;
        }
        match_columns += s_arg[i_arg].n_items - 1;
        break;
      case SET_EQUATE_COLUMNS:
        if (s_arg[i_arg].n_items < 2)
          SDDS_Bomb("Invalid -equate syntax.");
        equate_column = trealloc(equate_column, sizeof(*equate_column) * (equate_columns + s_arg[i_arg].n_items - 1));
        for (i = 1; i < s_arg[i_arg].n_items; i++) {
          if ((ptr = strchr(s_arg[i_arg].list[i], '=')))
            *ptr++ = 0;
          else
            ptr = s_arg[i_arg].list[i];
          equate_column[i - 1 + equate_columns][0] = s_arg[i_arg].list[i];
          equate_column[i - 1 + equate_columns][1] = ptr;
        }
        equate_columns += s_arg[i_arg].n_items - 1;
        break;
      case SET_REUSE:
        if (s_arg[i_arg].n_items == 1)
          reuse = 1;
        else {
          char *reuseOptions[2] = {"rows", "page"};
          for (i = 1; i < s_arg[i_arg].n_items; i++) {
            switch (match_string(s_arg[i_arg].list[i], reuseOptions, 2, 0)) {
            case 0:
              reuse = 1;
              break;
            case 1:
              reusePage = 1;
              break;
            default:
              SDDS_Bomb("Unknown reuse keyword.");
              break;
            }
          }
        }
        break;
      case SET_NOWARNINGS:
        warnings = 0;
        break;
      case SET_INVERT:
        invert = 1;
        break;
      case SET_PIPE:
        if (!processPipeOption(s_arg[i_arg].list + 1, s_arg[i_arg].n_items - 1, &pipeFlags))
          SDDS_Bomb("Invalid -pipe syntax.");
        break;
      default:
        fprintf(stderr, "Error: Unknown option: %s\n", s_arg[i_arg].list[0]);
        bomb(NULL, USAGE);
        break;
      }
    } else {
      if (input1 == NULL)
        input1 = s_arg[i_arg].list[0];
      else if (input2 == NULL)
        input2 = s_arg[i_arg].list[0];
      else if (output == NULL)
        output = s_arg[i_arg].list[0];
      else
        SDDS_Bomb("Too many filenames provided.");
    }
  }
 
  if (pipeFlags & USE_STDIN && input1) {
    if (output)
      SDDS_Bomb("Too many filenames with -pipe option.");
    output = input2;
    input2 = input1;
    input1 = NULL;
  }
  processFilenames("sddsmselect", &input1, &output, pipeFlags, !warnings, &tmpfile_used);
  if (!input2)
    SDDS_Bomb("Second input file not specified.");
 
  if (!match_columns && !equate_columns)
    SDDS_Bomb("Either -match or -equate must be specified.");
 
  if (!SDDS_InitializeInput(&SDDS_1, input1)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
  if (!SDDS_InitializeInput(&SDDS_2, input2)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
 
  for (i = 0; i < match_columns; i++) {
    if ((j = SDDS_GetColumnIndex(&SDDS_1, match_column[i][0])) < 0 ||
        SDDS_GetColumnType(&SDDS_1, j) != SDDS_STRING) {
      sprintf(s, "Error: Column '%s' not found or not of string type in file '%s'.",
              match_column[i][0], input1 ? input1 : "stdin");
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if ((j = SDDS_GetColumnIndex(&SDDS_2, match_column[i][1])) < 0 ||
        SDDS_GetColumnType(&SDDS_2, j) != SDDS_STRING) {
      sprintf(s, "Error: Column '%s' not found or not of string type in file '%s'.",
              match_column[i][1], input2);
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
  for (i = 0; i < equate_columns; i++) {
    if ((j = SDDS_GetColumnIndex(&SDDS_1, equate_column[i][0])) < 0 ||
        !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_1, j))) {
      sprintf(s, "Error: Column '%s' not found or not of numeric type in file '%s'.",
              equate_column[i][0], input1 ? input1 : "stdin");
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if ((j = SDDS_GetColumnIndex(&SDDS_2, equate_column[i][1])) < 0 ||
        !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_2, j))) {
      sprintf(s, "Error: Column '%s' not found or not of numeric type in file '%s'.",
              equate_column[i][1], input2);
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
 
  if (output && (pipeFlags & USE_STDOUT))
    SDDS_Bomb("Too many filenames with -pipe option.");
  if (!output && !(pipeFlags & USE_STDOUT)) {
    if (warnings)
      fprintf(stderr, "Warning: Existing file '%s' will be replaced.\n", input1 ? input1 : "stdin");
    tmpfile_used = 1;
    cp_str(&output, tmpname(NULL));
  }
  if (!SDDS_InitializeCopy(&SDDS_output, &SDDS_1, output, "w")) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
  if (columnMajorOrder != -1)
    SDDS_output.layout.data_mode.column_major = columnMajorOrder;
  else
    SDDS_output.layout.data_mode.column_major = SDDS_1.layout.data_mode.column_major;
  if (!SDDS_WriteLayout(&SDDS_output))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
 
  row_used = NULL;
  while ((retval1 = SDDS_ReadPage(&SDDS_1)) > 0) {
    if (!reusePage) {
      if ((retval2 = SDDS_ReadPage(&SDDS_2)) <= 0) {
        if (warnings)
          fprintf(stderr, "Warning: <input2> ends before <input1>.\n");
        if (invert) {
          /* Nothing to match, so everything would normally be thrown out */
          if (!SDDS_CopyPage(&SDDS_output, &SDDS_1) ||
              !SDDS_WritePage(&SDDS_output))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          continue;
        } else
          /* Nothing to match, so everything thrown out */
          break;
      }
    } else {
      if (retval1 == 1 && (retval2 = SDDS_ReadPage(&SDDS_2)) <= 0)
        SDDS_Bomb("<input2> has no data.");
      SDDS_SetRowFlags(&SDDS_2, 1);
    }
    SDDS_SetRowFlags(&SDDS_1, 1);
    rows1 = SDDS_CountRowsOfInterest(&SDDS_1);
    if ((rows2 = SDDS_CountRowsOfInterest(&SDDS_2))) {
      row_used = SDDS_Realloc(row_used, sizeof(*row_used) * rows2);
      SDDS_ZeroMemory(row_used, rows2 * sizeof(*row_used));
    }
    if (!SDDS_StartPage(&SDDS_output, rows1)) {
      SDDS_SetError("Problem starting output page.");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if (!SDDS_CopyParameters(&SDDS_output, &SDDS_1) ||
        !SDDS_CopyArrays(&SDDS_output, &SDDS_1)) {
      SDDS_SetError("Problem copying parameter or array data from first input file.");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    outputRow = 0;
    for (j = 0; j < rows1; j++) {
      /* Set up to match all rows of file 2 to row j of file 1 */
      SDDS_SetRowFlags(&SDDS_2, 1);
      for (i = 0; i < match_columns; i++) {
        if (!SDDS_GetValue(&SDDS_1, match_column[i][0], j, &match_value)) {
          sprintf(s, "Problem getting column '%s' from file '%s'.",
                  match_column[i][0], input1 ? input1 : "stdin");
          SDDS_SetError(s);
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
        if (SDDS_MatchRowsOfInterest(&SDDS_2, match_column[i][1], match_value, SDDS_AND) < 0) {
          sprintf(s, "Problem setting rows of interest for column '%s'.",
                  match_column[i][1]);
          SDDS_SetError(s);
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
      }
      n = SDDS_CountRowsOfInterest(&SDDS_2);
      if ((!n && !invert) || (n && invert))
        /* No match in file2 for row j of file1, or unwanted match found--so don't copy it */
        continue;
      for (k = 0; k < rows2; k++) {
        if (SDDS_GetRowFlag(&SDDS_2, k) < 0)
          /* Test if row k of file2 passed string-matches. If not, go to next row */
          continue;
        /* If row k of file2 is not already used, then test it for a match to row j of file1.
           If no -equate options were given, this test is always true.
        */
        if (!row_used[k]) {
          long equal;
          equal = rows_equate(&SDDS_1, j, &SDDS_2, k, equate_columns, equate_column);
          if ((equal && !invert) || (!equal && invert)) {
            row_used[k] = reuse ? 0 : 1;
            break;
          }
        }
      }
      if ((k == rows2 && !invert) || (k != rows2 && invert))
        /* No match in file2 for row j of file1, or unwanted match found--so don't copy it */
        continue;
      if (!SDDS_CopyRowDirect(&SDDS_output, outputRow, &SDDS_1, j)) {
        sprintf(s, "Problem copying to row %" PRId64 " of output from row %" PRId64 " of data set 1.",
                outputRow, j);
        SDDS_SetError(s);
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
      }
      outputRow++;
    }
    if (!SDDS_WritePage(&SDDS_output)) {
      SDDS_SetError("Problem writing data to output file.");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
 
  if (!SDDS_Terminate(&SDDS_1) ||
      !SDDS_Terminate(&SDDS_2) ||
      !SDDS_Terminate(&SDDS_output)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
 
  if (tmpfile_used && !replaceFileAndBackUp(input1, output))
    exit(EXIT_FAILURE);
 
  return EXIT_SUCCESS;
}

◆ rows_equate()

long rows_equate	(	SDDS_DATASET *	SDDS1,
		int64_t	row1,
		SDDS_DATASET *	SDDS2,
		int64_t	row2,
		long	equate_columns,
		STRING_PAIR *	equate_column )

Definition at line 404 of file sddsmselect.c.

                                                                                                                                        {
  char *data1, *data2;
  long index1, index2, size, type, i;
  char s[SDDS_MAXLINE];
  index2 = 0;
  for (i = 0; i < equate_columns; i++) {
    if ((index1 = SDDS_GetColumnIndex(SDDS1, equate_column[i][0])) < 0 ||
        (index2 = SDDS_GetColumnIndex(SDDS2, equate_column[i][1])) < 0) {
      SDDS_SetError("Problem equating rows.");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if ((type = SDDS_GetColumnType(SDDS1, index1)) != SDDS_GetColumnType(SDDS2, index2)) {
      sprintf(s, "Problem equating rows--types don't match for columns '%s' and '%s'.",
              equate_column[i][0], equate_column[i][1]);
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    size = SDDS_GetTypeSize(type);
    data1 = (char *)SDDS1->data[index1] + size * row1;
    data2 = (char *)SDDS2->data[index2] + size * row2;
    if (memcmp(data1, data2, size) != 0)
      return 0;
  }
  return 1;
}

Variable Documentation

◆ option

char* option[N_OPTIONS]

Initial value:

= {
  "match", 
  "equate", 
  "nowarnings", 
  "invert", 
  "reuse", 
  "pipe", 
  "majorOrder"
}

Definition at line 66 of file sddsmselect.c.

                          {
  "match", 
  "equate", 
  "nowarnings", 
  "invert", 
  "reuse", 
  "pipe", 
  "majorOrder"
};

◆ USAGE

char* USAGE