sddscheck.c File Reference

Detailed Description

Validates and checks an SDDS file for corruption or issues.

This program reads one or more SDDS (Self Describing Data Set) files and determines their validity. It processes each file by verifying its structure, pages, and data, and outputs the status:

• "ok" if the file is valid.
• "nonexistent" if the file does not exist.
• "badHeader" if the file has an invalid header.
• "corrupted" if the file contains errors. For multiple files, each output line is prefixed with the corresponding file name.

Usage

sddscheck [options] <filename|directory> [<filename|directory>...]

Options

Option	Description
-printErrors	Outputs detailed error messages to stderr.
-threads=<number>	Number of input files to check concurrently.
-summary	Prints aggregate counts after per-file output.
-failuresOnly	Suppresses ok results.
-failOnError	Exits nonzero if any checked file is not ok.
-recursive	Recursively expands directory arguments.
-pattern=<glob>	Filters directory expansion by file-name wildcard.
-maxErrors=<number>	Stops after the specified number of failed files.
-showPages	Adds the number of pages read to each result line.
-checkDefinitionsOnly	Validates only the SDDS header and definitions.
-verbose	Adds file size and SDDS layout metadata to each result line.
-stdin	Reads file or directory names from standard input.

Copyright

• (c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
• (c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.

License

This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.

Author

M. Borland, C. Saunders, R. Soliday

Definition in file sddscheck.c.

#include "mdb.h"
#include "SDDS.h"
#include "scan.h"
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>

Go to the source code of this file.

Functions
static void *	checkedMalloc (size_t size)
static void *	checkedRealloc (void *ptr, size_t size)
static char *	checkedStrdup (const char *text)
static int	parseLongOptionValue (const char *text, long minimum, long maximum, long *value)
static void	stringListAppend (StringList *list, const char *text)
static void	stringListFree (StringList *list)
static int	compareStringPointers (const void *left, const void *right)
static const char *	checkStatusString (CheckStatus status)
static const char *	dataModeString (int mode)
static const char *	baseName (const char *path)
static int	matchesPatterns (const char *path, const StringList *patterns)
static int	isDirectory (const char *path)
static char *	joinPath (const char *directory, const char *name)
static void	collectDirectoryFiles (const char *directory, long recursive, const StringList *patterns, StringList *files)
static void	readInputNamesFromStdin (StringList *input)
static CheckResult	checkFile (char *input, const CheckOptions *options, long print_errors)
static void	printCheckResult (const char *input, const CheckResult *result, long singleResult, long showPages, long verbose)
static void	printSummary (long checked, long *statusCounts)
int	main (int argc, char **argv)

Function Documentation

baseName()

static const char * baseName ( const char * path )

static

Definition at line 266 of file sddscheck.c.

                                              {
  const char *slash;
  const char *backslash;
 
  slash = strrchr(path, '/');
  backslash = strrchr(path, '\\');
  if (backslash && (!slash || backslash > slash))
    slash = backslash;
  return slash ? slash + 1 : path;
}

checkedMalloc()

static void * checkedMalloc ( size_t size )

static

Definition at line 173 of file sddscheck.c.

                                        {
  void *ptr;
  if (size == 0)
    size = 1;
  ptr = malloc(size);
  if (!ptr)
    SDDS_Bomb("memory allocation failure");
  return ptr;
}

checkedRealloc()

static void * checkedRealloc ( void * ptr, size_t size )

static

Definition at line 183 of file sddscheck.c.

                                                    {
  if (size == 0)
    size = 1;
  ptr = realloc(ptr, size);
  if (!ptr)
    SDDS_Bomb("memory allocation failure");
  return ptr;
}

checkedStrdup()

static char * checkedStrdup ( const char * text )

static

Definition at line 192 of file sddscheck.c.

                                             {
  char *copy;
  size_t length;
  if (!text)
    text = "";
  length = strlen(text) + 1;
  copy = checkedMalloc(length);
  memcpy(copy, text, length);
  return copy;
}

checkFile()

static CheckResult checkFile ( char * input, const CheckOptions * options, long print_errors )

static

Definition at line 409 of file sddscheck.c.

                                                                                          {
  SDDS_DATASET SDDS_input;
  CheckResult result;
  struct stat statBuffer;
  long retval;
 
  memset(&result, 0, sizeof(result));
  result.checked = 1;
  result.status = CHECK_CORRUPTED;
  result.dataMode = 0;
 
  if (stat(input, &statBuffer) != 0) {
    result.status = CHECK_NONEXISTENT;
    return result;
  }
  result.sizeKnown = 1;
  result.sizeBytes = (long long)statBuffer.st_size;
 
  if (!SDDS_InitializeInput(&SDDS_input, input)) {
    if (print_errors)
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    else
      SDDS_ClearErrors();
    result.status = CHECK_BADHEADER;
    return result;
  }
 
  result.metadataKnown = 1;
  result.dataMode = SDDS_input.layout.data_mode.mode;
  result.columns = SDDS_input.layout.n_columns;
  result.parameters = SDDS_input.layout.n_parameters;
  result.arrays = SDDS_input.layout.n_arrays;
  result.associates = SDDS_input.layout.n_associates;
 
  if (options->checkDefinitionsOnly) {
    result.status = CHECK_OK;
    SDDS_Terminate(&SDDS_input);
    return result;
  }
 
  while ((retval = SDDS_ReadPage(&SDDS_input)) > 0) {
    /* Loop continues until EOF or an error occurs. */
    result.pagesRead++;
  }
 
  if (retval == -1) {
    SDDS_Terminate(&SDDS_input);
    result.status = CHECK_OK;
    return result;
  }
 
  if (print_errors)
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
  else
    SDDS_ClearErrors();
  SDDS_Terminate(&SDDS_input);
  result.status = CHECK_CORRUPTED;
  return result;
}

checkStatusString()

static const char * checkStatusString ( CheckStatus status )

static

Definition at line 239 of file sddscheck.c.

                                                         {
  switch (status) {
  case CHECK_OK:
    return "ok";
  case CHECK_NONEXISTENT:
    return "nonexistent";
  case CHECK_BADHEADER:
    return "badHeader";
  case CHECK_CORRUPTED:
    return "corrupted";
  }
  return "corrupted";
}

collectDirectoryFiles()

static void collectDirectoryFiles ( const char * directory, long recursive, const StringList * patterns, StringList * files )

static

Definition at line 312 of file sddscheck.c.

                                                                                                                        {
  StringList entries = {NULL, 0, 0};
  long i;
 
#if SDDSCHECK_USE_DIRENT
  DIR *dir;
  struct dirent *entry;
 
  dir = opendir(directory);
  if (!dir) {
    fprintf(stderr, "warning: unable to read directory %s: %s\n", directory, strerror(errno));
    return;
  }
 
  while ((entry = readdir(dir))) {
    if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
      continue;
    stringListAppend(&entries, entry->d_name);
  }
  closedir(dir);
#elif defined(_WIN32)
  WIN32_FIND_DATAA entry;
  HANDLE findHandle;
  char *searchPattern;
  DWORD error;
 
  searchPattern = joinPath(directory, "*");
  findHandle = FindFirstFileA(searchPattern, &entry);
  if (findHandle == INVALID_HANDLE_VALUE) {
    error = GetLastError();
    if (error != ERROR_FILE_NOT_FOUND && error != ERROR_PATH_NOT_FOUND)
      fprintf(stderr, "warning: unable to read directory %s: Windows error %lu\n",
              directory, (unsigned long)error);
    free(searchPattern);
    return;
  }
 
  do {
    if (strcmp(entry.cFileName, ".") == 0 || strcmp(entry.cFileName, "..") == 0 ||
        (entry.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT))
      continue;
    stringListAppend(&entries, entry.cFileName);
  } while (FindNextFileA(findHandle, &entry));
  error = GetLastError();
  if (error != ERROR_NO_MORE_FILES)
    fprintf(stderr, "warning: unable to finish reading directory %s: Windows error %lu\n",
            directory, (unsigned long)error);
  FindClose(findHandle);
  free(searchPattern);
#else
  (void)directory;
  (void)recursive;
  (void)patterns;
  (void)files;
  SDDS_Bomb("-recursive and directory -pattern expansion are not supported on this platform");
#endif
 
  if (entries.items > 1)
    qsort(entries.item, entries.items, sizeof(*entries.item), compareStringPointers);
 
  for (i = 0; i < entries.items; i++) {
    char *path;
    struct stat statBuffer;
 
    path = joinPath(directory, entries.item[i]);
#if SDDSCHECK_USE_DIRENT
    if (lstat(path, &statBuffer) != 0) {
#else
    if (stat(path, &statBuffer) != 0) {
#endif
      free(path);
      continue;
    }
    if (S_ISDIR(statBuffer.st_mode)) {
      if (recursive)
        collectDirectoryFiles(path, recursive, patterns, files);
    } else if (S_ISREG(statBuffer.st_mode) && matchesPatterns(path, patterns)) {
      stringListAppend(files, path);
    }
    free(path);
  }
  stringListFree(&entries);
}

compareStringPointers()

static int compareStringPointers ( const void * left, const void * right )

static

Definition at line 233 of file sddscheck.c.

                                                                      {
  const char *const *leftString = left;
  const char *const *rightString = right;
  return strcmp(*leftString, *rightString);
}

dataModeString()

static const char * dataModeString ( int mode )

static

Definition at line 253 of file sddscheck.c.

                                            {
  switch (mode) {
  case SDDS_BINARY:
    return "binary";
  case SDDS_ASCII:
    return "ascii";
  case SDDS_PARALLEL:
    return "parallel";
  default:
    return "unknown";
  }
}

isDirectory()

static int isDirectory ( const char * path )

static

Definition at line 291 of file sddscheck.c.

                                         {
  struct stat statBuffer;
  if (stat(path, &statBuffer) != 0)
    return 0;
  return S_ISDIR(statBuffer.st_mode);
}

joinPath()

static char * joinPath ( const char * directory, const char * name )

static

Definition at line 298 of file sddscheck.c.

                                                               {
  char *path;
  size_t directoryLength, nameLength, length;
  int needsSlash;
 
  directoryLength = strlen(directory);
  nameLength = strlen(name);
  needsSlash = directoryLength && directory[directoryLength - 1] != '/' && directory[directoryLength - 1] != '\\';
  length = directoryLength + needsSlash + nameLength + 1;
  path = checkedMalloc(length);
  snprintf(path, length, "%s%s%s", directory, needsSlash ? "/" : "", name);
  return path;
}

main()

int main	(	int	argc,
		char **	argv )

Definition at line 494 of file sddscheck.c.

                                {
  StringList rawInput = {NULL, 0, 0};
  StringList input = {NULL, 0, 0};
  StringList patterns = {NULL, 0, 0};
  CheckOptions checkOptions;
  CheckResult *result;
  long i_arg, print_errors, threads, summary, failuresOnly, failOnError, recursive, useStdin, maxErrors, showPages, verbose;
  long checked, failures, statusCounts[4];
  SCANNED_ARG *s_arg;
 
  /* Register the program name for error reporting. */
  SDDS_RegisterProgramName(argv[0]);
 
  /* Parse command-line arguments. */
  argc = scanargs(&s_arg, argc, argv);
  if (!s_arg || argc < 2) {
    bomb(NULL, usage); /* Display usage and exit if arguments are insufficient. */
  }
 
  memset(&checkOptions, 0, sizeof(checkOptions));
  memset(statusCounts, 0, sizeof(statusCounts));
  print_errors = 0;
  threads = 1;
  summary = 0;
  failuresOnly = 0;
  failOnError = 0;
  recursive = 0;
  useStdin = 0;
  maxErrors = 0;
  showPages = 0;
  verbose = 0;
 
  /* Process each command-line argument. */
  for (i_arg = 1; i_arg < argc; i_arg++) {
    if (s_arg[i_arg].arg_type == OPTION) {
      /* Match recognized options. */
      switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
      case CLO_PRINTERRORS:
        print_errors = 1;
        break;
      case CLO_THREADS:
        if (s_arg[i_arg].n_items != 2 ||
            !parseLongOptionValue(s_arg[i_arg].list[1], 1, INT_MAX, &threads))
          SDDS_Bomb("invalid -threads syntax");
        break;
      case CLO_SUMMARY:
        summary = 1;
        break;
      case CLO_FAILURESONLY:
        failuresOnly = 1;
        break;
      case CLO_FAILONERROR:
        failOnError = 1;
        break;
      case CLO_RECURSIVE:
        recursive = 1;
        break;
      case CLO_PATTERN:
        if (s_arg[i_arg].n_items != 2 || !strlen(s_arg[i_arg].list[1]))
          SDDS_Bomb("invalid -pattern syntax");
        stringListAppend(&patterns, s_arg[i_arg].list[1]);
        break;
      case CLO_MAXERRORS:
        if (s_arg[i_arg].n_items != 2 ||
            !parseLongOptionValue(s_arg[i_arg].list[1], 1, LONG_MAX, &maxErrors))
          SDDS_Bomb("invalid -maxErrors syntax");
        break;
      case CLO_SHOWPAGES:
        showPages = 1;
        break;
      case CLO_CHECKDEFINITIONSONLY:
        checkOptions.checkDefinitionsOnly = 1;
        break;
      case CLO_VERBOSE:
        verbose = 1;
        break;
      case CLO_STDIN:
        useStdin = 1;
        break;
      default:
        SDDS_Bomb("unknown option given"); /* Handle unrecognized options. */
        break;
      }
    } else {
      stringListAppend(&rawInput, s_arg[i_arg].list[0]);
    }
  }
 
  if (useStdin)
    readInputNamesFromStdin(&rawInput);
 
  if (rawInput.items < 1)
    bomb(NULL, usage);
 
  for (i_arg = 0; i_arg < rawInput.items; i_arg++) {
    if ((recursive || patterns.items) && isDirectory(rawInput.item[i_arg]))
      collectDirectoryFiles(rawInput.item[i_arg], recursive, &patterns, &input);
    else
      stringListAppend(&input, rawInput.item[i_arg]);
  }
 
  result = checkedMalloc(sizeof(*result) * input.items);
  memset(result, 0, sizeof(*result) * input.items);
  checked = 0;
  failures = 0;
 
  if (input.items == 1 && !summary && !failuresOnly && !showPages && !verbose) {
    result[0] = checkFile(input.item[0], &checkOptions, 0);
    checked = 1;
    statusCounts[result[0].status]++;
    failures = result[0].status != CHECK_OK;
    if (print_errors && result[0].status == CHECK_CORRUPTED)
      checkFile(input.item[0], &checkOptions, 1);
    printCheckResult(input.item[0], &result[0], 1, showPages, verbose);
    if (print_errors && result[0].status == CHECK_BADHEADER)
      checkFile(input.item[0], &checkOptions, 1);
    free(result);
    stringListFree(&patterns);
    stringListFree(&rawInput);
    stringListFree(&input);
    return (failOnError && failures) ? 1 : 0;
  }
 
  if (threads <= 1 || input.items <= 1) {
    for (i_arg = 0; i_arg < input.items; i_arg++) {
      result[i_arg] = checkFile(input.item[i_arg], &checkOptions, print_errors);
      checked++;
      statusCounts[result[i_arg].status]++;
      if (result[i_arg].status != CHECK_OK) {
        failures++;
        if (maxErrors && failures >= maxErrors)
          break;
      }
    }
  } else {
    long sharedFailures = 0;
    if (threads > input.items)
      threads = input.items;
#if SDDSCHECK_USE_OPENMP
    omp_set_num_threads((int)threads);
#pragma omp parallel for schedule(dynamic)
    for (i_arg = 0; i_arg < input.items; i_arg++) {
      long failuresSeen = 0;
      if (maxErrors) {
#pragma omp critical(sddscheck_failures)
        {
          failuresSeen = sharedFailures;
        }
        if (failuresSeen >= maxErrors)
          continue;
      }
      result[i_arg] = checkFile(input.item[i_arg], &checkOptions, 0);
      if (maxErrors && result[i_arg].status != CHECK_OK) {
#pragma omp critical(sddscheck_failures)
        {
          sharedFailures++;
        }
      }
    }
#else
    for (i_arg = 0; i_arg < input.items; i_arg++) {
      if (maxErrors && sharedFailures >= maxErrors)
        break;
      result[i_arg] = checkFile(input.item[i_arg], &checkOptions, 0);
      if (result[i_arg].status != CHECK_OK)
        sharedFailures++;
    }
#endif
    if (print_errors) {
      for (i_arg = 0; i_arg < input.items; i_arg++) {
        if (result[i_arg].checked && result[i_arg].status != CHECK_OK && result[i_arg].status != CHECK_NONEXISTENT)
          checkFile(input.item[i_arg], &checkOptions, 1);
      }
    }
    for (i_arg = 0; i_arg < input.items; i_arg++) {
      if (!result[i_arg].checked)
        continue;
      checked++;
      statusCounts[result[i_arg].status]++;
      if (result[i_arg].status != CHECK_OK)
        failures++;
    }
  }
 
  for (i_arg = 0; i_arg < input.items; i_arg++) {
    if (!result[i_arg].checked)
      continue;
    if (failuresOnly && result[i_arg].status == CHECK_OK)
      continue;
    printCheckResult(input.item[i_arg], &result[i_arg], input.items == 1, showPages, verbose);
  }
  if (summary)
    printSummary(checked, statusCounts);
 
  free(result);
  stringListFree(&patterns);
  stringListFree(&rawInput);
  stringListFree(&input);
  return (failOnError && failures) ? 1 : 0;
}

matchesPatterns()

static int matchesPatterns ( const char * path, const StringList * patterns )

static

Definition at line 277 of file sddscheck.c.

                                                                         {
  long i;
  const char *name;
 
  if (!patterns->items)
    return 1;
  name = baseName(path);
  for (i = 0; i < patterns->items; i++) {
    if (wild_match((char *)name, patterns->item[i]))
      return 1;
  }
  return 0;
}

parseLongOptionValue()

static int parseLongOptionValue ( const char * text, long minimum, long maximum, long * value )

static

Definition at line 203 of file sddscheck.c.

                                                                                           {
  char *endptr;
  long parsed;
 
  if (!text || !*text)
    return 0;
  errno = 0;
  parsed = strtol(text, &endptr, 10);
  if (errno == ERANGE || endptr == text || *endptr || parsed < minimum || parsed > maximum)
    return 0;
  *value = parsed;
  return 1;
}

printCheckResult()

static void printCheckResult ( const char * input, const CheckResult * result, long singleResult, long showPages, long verbose )

static

Definition at line 469 of file sddscheck.c.

                                                                                                                            {
  if (singleResult && !showPages && !verbose) {
    puts(checkStatusString(result->status));
    return;
  }
 
  printf("%s: %s", input, checkStatusString(result->status));
  if (showPages || verbose)
    printf(" pagesRead=%ld", result->pagesRead);
  if (verbose) {
    if (result->sizeKnown)
      printf(" sizeBytes=%lld", result->sizeBytes);
    if (result->metadataKnown) {
      printf(" dataMode=%s columns=%d parameters=%d arrays=%d associates=%d",
             dataModeString(result->dataMode), result->columns, result->parameters, result->arrays, result->associates);
    }
  }
  putchar('\n');
}

printSummary()

static void printSummary ( long checked, long * statusCounts )

static

Definition at line 489 of file sddscheck.c.

                                                           {
  printf("checked: %ld ok: %ld nonexistent: %ld badHeader: %ld corrupted: %ld\n",
         checked, statusCounts[CHECK_OK], statusCounts[CHECK_NONEXISTENT], statusCounts[CHECK_BADHEADER], statusCounts[CHECK_CORRUPTED]);
}

readInputNamesFromStdin()

static void readInputNamesFromStdin ( StringList * input )

static

Definition at line 396 of file sddscheck.c.

                                                       {
  char buffer[4096];
 
  while (fgets(buffer, sizeof(buffer), stdin)) {
    size_t length = strlen(buffer);
    while (length && (buffer[length - 1] == '\n' || buffer[length - 1] == '\r'))
      buffer[--length] = 0;
    if (!length)
      continue;
    stringListAppend(input, buffer);
  }
}

stringListAppend()

static void stringListAppend ( StringList * list, const char * text )

static

Definition at line 217 of file sddscheck.c.

                                                                 {
  if (list->items >= list->allocated) {
    list->allocated = list->allocated ? 2 * list->allocated : 16;
    list->item = checkedRealloc(list->item, sizeof(*list->item) * list->allocated);
  }
  list->item[list->items++] = checkedStrdup(text);
}

stringListFree()

static void stringListFree ( StringList * list )

static

Definition at line 225 of file sddscheck.c.

                                             {
  long i;
  for (i = 0; i < list->items; i++)
    free(list->item[i]);
  free(list->item);
  memset(list, 0, sizeof(*list));
}