Creates an SDDS data set from another data set based on matching data in a third data set. More...

#include "mdb.h"
#include "SDDS.h"
#include "scan.h"

Enumerations
enum	option_type { SET_MATCH_COLUMN , SET_EQUATE_COLUMN , SET_NOWARNINGS , SET_INVERT , SET_REUSE , SET_PIPE , SET_MAJOR_ORDER , N_OPTIONS }

Functions
int	main (int argc, char **argv)

Variables
char *	option [N_OPTIONS]

char *	USAGE

Detailed Description

Creates an SDDS data set from another data set based on matching data in a third data set.

This program selects rows from <input1> that have (or do not have, if inverted) a matching entry in <input2>, based on specified matching or equating columns. The output is written to <output>, or <input1> is replaced if <output> is not provided.

Usage

sddsselect [OPTIONS] [<input1>] <input2> [<output>]

Options

-pipe[=input][,output] Use pipe for input and/or output.
-match=<column1>[=<column2>] Specify columns to match between input1 and input2.
-equate=<column1>[=<column2>] Specify columns to equate between input1 and input2.
-invert Invert the selection to keep non-matching rows.
-reuse[=rows][,page] Allow reusing rows or specify page reuse.
-majorOrder=row|column Set the output file to row or column major order.
-nowarnings Suppress warning messages.

Copyright

(c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
(c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.

License: This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.

Author: M. Borland, C. Saunders, R. Soliday, H. Shang

Definition in file sddsselect.c.

Enumeration Type Documentation

◆ option_type

enum option_type

Definition at line 52 of file sddsselect.c.

                 {
  SET_MATCH_COLUMN,
  SET_EQUATE_COLUMN,
  SET_NOWARNINGS,
  SET_INVERT,
  SET_REUSE,
  SET_PIPE,
  SET_MAJOR_ORDER,
  N_OPTIONS
};

Function Documentation

◆ main()

int main	(	int	argc,
		char **	argv )

Definition at line 89 of file sddsselect.c.

                                {
  SDDS_DATASET SDDS_1, SDDS_2, SDDS_output;
  long i, j, i_arg, reuse, reusePage;
  int64_t rows1, rows2, i1, i2;
  SCANNED_ARG *s_arg;
  char s[200], *ptr;
  char **match_column, **equate_column;
  long match_columns, equate_columns;
  char *input1, *input2, *output;
  long tmpfile_used, retval1, retval2;
  long warnings, invert;
  unsigned long pipeFlags, majorOrderFlag;
  KEYED_EQUIVALENT **keyGroup = NULL;
  long keyGroups = 0;
  short columnMajorOrder = -1;
 
  SDDS_RegisterProgramName(argv[0]);
  argc = scanargs(&s_arg, argc, argv);
  if (argc < 3)
    bomb(NULL, USAGE);
 
  input1 = input2 = output = NULL;
  match_column = equate_column = NULL;
  match_columns = equate_columns = reuse = reusePage = 0;
  tmpfile_used = invert = 0;
  warnings = 1;
  pipeFlags = 0;
 
  for (i_arg = 1; i_arg < argc; i_arg++) {
    if (s_arg[i_arg].arg_type == OPTION) {
      delete_chars(s_arg[i_arg].list[0], "_");
      switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
      case SET_MAJOR_ORDER:
        majorOrderFlag = 0;
        s_arg[i_arg].n_items--;
        if (s_arg[i_arg].n_items > 0 &&
            (!scanItemList(&majorOrderFlag, s_arg[i_arg].list + 1, &s_arg[i_arg].n_items, 0,
                           "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
                           "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL)))
          SDDS_Bomb("invalid -majorOrder syntax/values");
        if (majorOrderFlag & SDDS_COLUMN_MAJOR_ORDER)
          columnMajorOrder = 1;
        else if (majorOrderFlag & SDDS_ROW_MAJOR_ORDER)
          columnMajorOrder = 0;
        break;
      case SET_MATCH_COLUMN:
        if (s_arg[i_arg].n_items != 2)
          SDDS_Bomb("invalid -match syntax");
        if (match_columns != 0)
          SDDS_Bomb("only one -match option may be given");
        match_column = tmalloc(sizeof(*match_column) * 2);
        if ((ptr = strchr(s_arg[i_arg].list[1], '=')))
          *ptr++ = 0;
        else
          ptr = s_arg[i_arg].list[1];
        match_column[0] = s_arg[i_arg].list[1];
        match_column[1] = ptr;
        match_columns = 1;
        break;
      case SET_EQUATE_COLUMN:
        if (s_arg[i_arg].n_items != 2)
          SDDS_Bomb("invalid -equate syntax");
        if (equate_columns != 0)
          SDDS_Bomb("only one -equate option may be given");
        equate_column = tmalloc(sizeof(*equate_column) * 2);
        if ((ptr = strchr(s_arg[i_arg].list[1], '=')))
          *ptr++ = 0;
        else
          ptr = s_arg[i_arg].list[1];
        equate_column[0] = s_arg[i_arg].list[1];
        equate_column[1] = ptr;
        equate_columns = 1;
        break;
      case SET_REUSE:
        if (s_arg[i_arg].n_items == 1)
          reuse = 1;
        else {
          char *reuseOptions[2] = {"rows", "page"};
          for (i = 1; i < s_arg[i_arg].n_items; i++) {
            switch (match_string(s_arg[i_arg].list[i], reuseOptions, 2, 0)) {
            case 0:
              reuse = 1;
              break;
            case 1:
              reusePage = 1;
              break;
            default:
              SDDS_Bomb("unknown reuse keyword");
              break;
            }
          }
        }
        break;
      case SET_INVERT:
        invert = 1;
        break;
      case SET_NOWARNINGS:
        warnings = 0;
        break;
      case SET_PIPE:
        if (!processPipeOption(s_arg[i_arg].list + 1, s_arg[i_arg].n_items - 1, &pipeFlags))
          SDDS_Bomb("invalid -pipe syntax");
        break;
      default:
        fprintf(stderr, "error: unknown switch: %s\n", s_arg[i_arg].list[0]);
        SDDS_Bomb(NULL);
        break;
      }
    } else {
      if (input1 == NULL)
        input1 = s_arg[i_arg].list[0];
      else if (input2 == NULL)
        input2 = s_arg[i_arg].list[0];
      else if (output == NULL)
        output = s_arg[i_arg].list[0];
      else
        SDDS_Bomb("too many filenames");
    }
  }
 
  if (pipeFlags & USE_STDIN && input1) {
    if (output)
      SDDS_Bomb("too many filenames (sddsxref)");
    output = input2;
    input2 = input1;
    input1 = NULL;
  }
  processFilenames("sddsselect", &input1, &output, pipeFlags, !warnings, &tmpfile_used);
  if (!input2)
    SDDS_Bomb("second input file not specified (sddsxref)");
 
  if (equate_columns && match_columns)
    SDDS_Bomb("only one of -equate or -match may be given");
  if (!equate_columns && !match_columns)
    SDDS_Bomb("one of -equate or -match must be given");
 
  if (!SDDS_InitializeInput(&SDDS_1, input1)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
  if (!SDDS_InitializeInput(&SDDS_2, input2)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
 
  if (match_columns) {
    if ((j = SDDS_GetColumnIndex(&SDDS_1, match_column[0])) < 0 ||
        SDDS_GetColumnType(&SDDS_1, j) != SDDS_STRING) {
      sprintf(s, "error: column %s not found or not string type in file %s", match_column[0], input1 ? input1 : "stdin");
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if ((j = SDDS_GetColumnIndex(&SDDS_2, match_column[1])) < 0 ||
        SDDS_GetColumnType(&SDDS_2, j) != SDDS_STRING) {
      sprintf(s, "error: column %s not found or not string type in file %s", match_column[1], input2);
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
  if (equate_columns) {
    if ((j = SDDS_GetColumnIndex(&SDDS_1, equate_column[0])) < 0 ||
        !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_1, j))) {
      sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[0], input1 ? input1 : "stdin");
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if ((j = SDDS_GetColumnIndex(&SDDS_2, equate_column[1])) < 0 ||
        !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_2, j))) {
      sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[1], input2);
      SDDS_SetError(s);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
 
  if (output && pipeFlags & USE_STDOUT)
    SDDS_Bomb("too many filenames with -pipe option");
  if (!output && !(pipeFlags & USE_STDOUT)) {
    if (warnings)
      fprintf(stderr, "warning: existing file %s will be replaced (sddsselect)\n", input1 ? input1 : "stdin");
    tmpfile_used = 1;
    cp_str(&output, tmpname(NULL));
  }
  if (!SDDS_InitializeCopy(&SDDS_output, &SDDS_1, output, "w")) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
  if (columnMajorOrder != -1)
    SDDS_output.layout.data_mode.column_major = columnMajorOrder;
  else
    SDDS_output.layout.data_mode.column_major = SDDS_1.layout.data_mode.column_major;
  if (!SDDS_WriteLayout(&SDDS_output))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
 
  while ((retval1 = SDDS_ReadPage(&SDDS_1)) > 0) {
    if (!reusePage) {
      if ((retval2 = SDDS_ReadPage(&SDDS_2)) <= 0) {
        if (warnings)
          fprintf(stderr, "warning: <input2> ends before <input1>\n");
        if (invert) {
          /* nothing to match, so everything would normally be thrown out */
          if (!SDDS_CopyPage(&SDDS_output, &SDDS_1) ||
              !SDDS_WritePage(&SDDS_output))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          continue;
        } else
          /* nothing to match, so everything thrown out */
          break;
      }
    } else {
      if (retval1 == 1 && (retval2 = SDDS_ReadPage(&SDDS_2)) <= 0)
        SDDS_Bomb("<input2> has no data");
      SDDS_SetRowFlags(&SDDS_2, 1);
    }
    rows1 = SDDS_CountRowsOfInterest(&SDDS_1);
    rows2 = SDDS_CountRowsOfInterest(&SDDS_2);
 
    if (!SDDS_StartPage(&SDDS_output, rows1)) {
      SDDS_SetError("Problem starting output page");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if (!SDDS_CopyParameters(&SDDS_output, &SDDS_2) ||
        !SDDS_CopyArrays(&SDDS_output, &SDDS_2)) {
      SDDS_SetError("Problem copying parameter or array data from second input file");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if (!SDDS_CopyParameters(&SDDS_output, &SDDS_1) ||
        !SDDS_CopyArrays(&SDDS_output, &SDDS_1)) {
      SDDS_SetError("Problem copying parameter or array data from first input file");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
    if (rows1) {
      if (match_columns) {
        char **string1, **string2;
        long matched;
        string2 = NULL;
        if (!(string1 = SDDS_GetColumn(&SDDS_1, match_column[0]))) {
          fprintf(stderr, "Error: problem getting column %s from file %s\n", match_column[0], input1 ? input1 : "stdin");
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
        if (rows2 && !(string2 = SDDS_GetColumn(&SDDS_2, match_column[1]))) {
          fprintf(stderr, "Error: problem getting column %s from file %s\n", match_column[1], input2);
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
        if (rows2)
          keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_STRING, string2, rows2);
        for (i1 = 0; i1 < rows1; i1++) {
          if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
            sprintf(s, "Problem copying row %" PRId64 " of first data set", i1);
            SDDS_SetError(s);
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          }
          matched = 0;
          if (rows2 && (i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_STRING, string1 + i1, reuse)) >= 0) {
            matched = 1;
          }
          if ((!matched && !invert) || (matched && invert)) {
            if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
              SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          }
        }
        if (string1) {
          for (i = 0; i < rows1; i++)
            free(string1[i]);
          free(string1);
          string1 = NULL;
        }
        if (string2) {
          for (i = 0; i < rows2; i++)
            free(string2[i]);
          free(string2);
          string2 = NULL;
        }
        for (i = 0; i < keyGroups; i++) {
          if (keyGroup[i]) {
            if (keyGroup[i]->equivalent)
              free(keyGroup[i]->equivalent);
            free(keyGroup[i]);
            keyGroup[i] = NULL;
          }
        }
        if (keyGroups) {
          free(keyGroup);
          keyGroup = NULL;
          keyGroups = 0;
        }
      } else if (equate_columns) {
        double *value1, *value2;
        long equated;
        value2 = NULL;
        if (!(value1 = SDDS_GetColumnInDoubles(&SDDS_1, equate_column[0]))) {
          fprintf(stderr, "Error: problem getting column %s from file %s\n", equate_column[0], input1 ? input1 : "stdin");
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
        if (rows2 && !(value2 = SDDS_GetColumnInDoubles(&SDDS_2, equate_column[1]))) {
          fprintf(stderr, "Error: problem getting column %s from file %s\n", equate_column[1], input2);
          SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
        }
        if (rows2)
          keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_DOUBLE, value2, rows2);
        for (i1 = 0; i1 < rows1; i1++) {
          if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
            sprintf(s, "Problem copying row %" PRId64 " of first data set", i1);
            SDDS_SetError(s);
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          }
          equated = 0;
          if (rows2 &&
              (i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_DOUBLE, value1 + i1, reuse)) >= 0) {
            equated = 1;
          }
          if ((!equated && !invert) || (equated && invert)) {
            if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
              SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
          }
        }
        if (value1)
          free(value1);
        value1 = NULL;
        if (rows2 && value2)
          free(value2);
        value2 = NULL;
        for (i = 0; i < keyGroups; i++) {
          if (keyGroup[i]) {
            if (keyGroup[i]->equivalent)
              free(keyGroup[i]->equivalent);
            free(keyGroup[i]);
            keyGroup[i] = NULL;
          }
        }
        if (keyGroups) {
          free(keyGroup);
          keyGroup = NULL;
          keyGroups = 0;
        }
      }
    }
    if (!SDDS_WritePage(&SDDS_output)) {
      SDDS_SetError("Problem writing data to output file");
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
    }
  }
 
  if (!SDDS_Terminate(&SDDS_1) || !SDDS_Terminate(&SDDS_2) || !SDDS_Terminate(&SDDS_output)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(EXIT_FAILURE);
  }
  if (tmpfile_used && !replaceFileAndBackUp(input1, output))
    exit(EXIT_FAILURE);
  free_scanargs(&s_arg, argc);
  if (match_columns)
    free(match_column);
  return EXIT_SUCCESS;
}

Variable Documentation

◆ option

char* option[N_OPTIONS]

Initial value:

= {
  "match",
  "equate",
  "nowarnings",
  "invert",
  "reuse",
  "pipe",
  "majorOrder",
}

Definition at line 63 of file sddsselect.c.

                          {
  "match",
  "equate",
  "nowarnings",
  "invert",
  "reuse",
  "pipe",
  "majorOrder",
};

◆ USAGE

char* USAGE