SDDS ToolKit Programs and Libraries for C and Python
All Classes Files Functions Variables Macros Pages
sddscombine.c File Reference

Detailed Description

Combines multiple SDDS files into a single SDDS file with options for merging, appending, and modifying data.

This program processes multiple SDDS files, allowing users to merge data, append new pages, or perform transformations such as deleting or retaining specific columns, parameters, or arrays. It supports various modes for handling sparse data, recovery of corrupted data, and control over the order of the output dataset.

Usage

sddscombine [<SDDSinputfilelist>] [<SDDSoutputfile>]
[-pipe=[input][,output]]
[-delete={column|parameter|array},<matching-string>[,...]]
[-retain={column|parameter|array},<matching-string>[,...]]
[-sparse=<integer>[,{average|median|minimum|maximum}]]
[-merge[=<parameter-name>|<npages>]]
[-append]
[-overWrite]
[-collapse]
[-recover[=clip]]
[-majorOrder=row|column]

Options

Option Description
-pipe Enable piping for input and/or output.
-delete Delete columns, parameters, or arrays matching the specified pattern.
-retain Retain only columns, parameters, or arrays matching the specified pattern.
-sparse Sample every nth row, optionally performing statistical analysis (average, median, etc.).
-merge Merge pages based on a parameter or number of pages.
-append Append data to the first input file.
-overWrite Overwrite the output file if it exists.
-collapse Collapse the output into a single page, as processed through sddscollapse.
-recover Recover incomplete or corrupted data, optionally clipping incomplete pages.
-majorOrder Specify row-major or column-major order for output.

Incompatibilities

  • -collapse is incompatible with:
    • -append
  • Only one of the following may be specified:
    • -delete
    • -retain
  • For -merge:
    • <parameter-name> must exist in all input files if specified.
License
This file is distributed under the terms of the Software License Agreement found in the file LICENSE included with this distribution.
Authors
M. Borland, C. Saunders, R. Soliday, H. Shang

Definition in file sddscombine.c.

#include "mdb.h"
#include "scan.h"
#include "SDDS.h"

Go to the source code of this file.

Functions

long SDDS_CompareParameterValues (void *param1, void *param2, long type)
 
long keep_element (char *name, char **delete, long deletions, char **retain, long retentions)
 
void * SDDS_GetParameterMod (SDDS_DATASET *SDDS_dataset, SDDS_DATASET *SDDS_output, char *parameter_name, void *memory)
 
int main (int argc, char **argv)
 

Function Documentation

◆ keep_element()

long keep_element ( char * name,
char ** delete,
long deletions,
char ** retain,
long retentions )

Definition at line 907 of file sddscombine.c.

907 {
908 long i, flag;
909
910 flag = 1;
911
912 if (deletions) {
913 for (i = 0; i < deletions; i++) {
914 if (wild_match(name, delete[i])) {
915 flag = 0;
916 break;
917 }
918 }
919 }
920
921 if (retentions) {
922 if (!deletions)
923 flag = 0;
924 for (i = 0; i < retentions; i++) {
925 if (wild_match(name, retain[i])) {
926 flag = 1;
927 break;
928 }
929 }
930 }
931
932 return flag;
933}
int wild_match(char *string, char *template)
Determine whether one string is a wildcard match for another.
Definition wild_match.c:49

◆ main()

int main ( int argc,
char ** argv )

Definition at line 149 of file sddscombine.c.

149 {
150 SDDS_DATASET SDDS_input, SDDS_output;
151 char **inputfile, *outputfile;
152 long inputfiles, i, i_arg, retval = 0, first_page;
153 long iFile, first_data, sparse, setPageNumber;
154 int32_t sparse_statistics = 0;
155 long merge, nMerge, overwrite, collapse, page, append;
156 int64_t allocated_rows;
157 int32_t columns;
158 SCANNED_ARG *s_arg;
159 long buffer[16];
160 char *param, *last_param, *this_param, *text, *contents, **column;
161 long param_index, param_type, param_size, output_pending;
162 unsigned long pipeFlags, majorOrderFlag;
163
164 char **retain_column, **delete_column;
165 long retain_columns, delete_columns;
166 char **retain_parameter, **delete_parameter;
167 long retain_parameters, delete_parameters;
168 char **retain_array, **delete_array;
169 long retain_arrays, delete_arrays;
170 long nColumns, recover, recovered;
171 short columnMajorOrder = -1;
172
174 argc = scanargs(&s_arg, argc, argv);
175 if (argc < 3)
176 bomb(NULL, USAGE);
177
178 setPageNumber = allocated_rows = param_type = param_size = 0;
179 last_param = this_param = NULL;
180 column = NULL;
181 inputfile = NULL;
182 outputfile = param = NULL;
183 inputfiles = merge = overwrite = collapse = append = nMerge = 0;
184 pipeFlags = 0;
185 sparse = 1;
186 recover = 0;
187
188 retain_column = delete_column = NULL;
189 retain_columns = delete_columns = 0;
190 retain_parameter = delete_parameter = NULL;
191 retain_parameters = delete_parameters = 0;
192 retain_array = delete_array = NULL;
193 retain_arrays = delete_arrays = 0;
194
196 argc = scanargs(&s_arg, argc, argv);
197 if (argc < 3)
198 bomb(NULL, USAGE);
199
200 for (i_arg = 1; i_arg < argc; i_arg++) {
201 if (s_arg[i_arg].arg_type == OPTION) {
202 switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
203 case SET_MAJOR_ORDER:
204 majorOrderFlag = 0;
205 s_arg[i_arg].n_items -= 1;
206 if (s_arg[i_arg].n_items > 0 &&
207 (!scanItemList(&majorOrderFlag, s_arg[i_arg].list + 1, &s_arg[i_arg].n_items, 0,
208 "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
209 "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL)))
210 SDDS_Bomb("invalid -majorOrder syntax/values");
211 if (majorOrderFlag & SDDS_COLUMN_MAJOR_ORDER)
212 columnMajorOrder = 1;
213 else if (majorOrderFlag & SDDS_ROW_MAJOR_ORDER)
214 columnMajorOrder = 0;
215 break;
216 case SET_MERGE:
217 if (s_arg[i_arg].n_items > 2)
218 bomb("invalid -merge syntax", USAGE);
219 merge = 1;
220 param = NULL;
221 nMerge = -1;
222 if (s_arg[i_arg].n_items == 2) {
223 if (isdigit(s_arg[i_arg].list[1][0])) {
224 if (!sscanf(s_arg[i_arg].list[1], "%ld", &nMerge))
225 bomb("invalid -merge syntax (could not scan number of pages)", USAGE);
226 } else
227 param = s_arg[i_arg].list[1];
228 }
229 break;
230 case SET_APPEND:
231 if (s_arg[i_arg].n_items > 1)
232 bomb("invalid -append syntax", USAGE);
233 append = 1;
234 if (collapse) {
235 SDDS_Bomb("-collapse and -append options cannot be used together");
236 break;
237 }
238 break;
239 case SET_OVERWRITE:
240 overwrite = 1;
241 break;
242 case SET_PIPE:
243 if (!processPipeOption(s_arg[i_arg].list + 1, s_arg[i_arg].n_items - 1, &pipeFlags))
244 SDDS_Bomb("invalid -pipe syntax");
245 break;
246 case SET_RECOVER:
247 recover = 1;
248 if (s_arg[i_arg].n_items != 1) {
249 recover = 2;
250 if (s_arg[i_arg].n_items > 2 || strncmp(s_arg[i_arg].list[1], "clip", strlen(s_arg[i_arg].list[1])) != 0)
251 SDDS_Bomb("invalid -recover syntax");
252 }
253 break;
254 case SET_DELETE:
255 if (s_arg[i_arg].n_items < 3)
256 SDDS_Bomb("invalid -delete syntax");
257 switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
258 case COLUMN_MODE:
259 delete_column = trealloc(delete_column, sizeof(*delete_column) * (delete_columns + s_arg[i_arg].n_items - 2));
260 for (i = 2; i < s_arg[i_arg].n_items; i++)
261 delete_column[i - 2 + delete_columns] = expand_ranges(s_arg[i_arg].list[i]);
262 delete_columns += s_arg[i_arg].n_items - 2;
263 break;
264 case PARAMETER_MODE:
265 delete_parameter = trealloc(delete_parameter, sizeof(*delete_parameter) * (delete_parameters + s_arg[i_arg].n_items - 2));
266 for (i = 2; i < s_arg[i_arg].n_items; i++)
267 delete_parameter[i - 2 + delete_parameters] = expand_ranges(s_arg[i_arg].list[i]);
268 delete_parameters += s_arg[i_arg].n_items - 2;
269 break;
270 case ARRAY_MODE:
271 delete_array = trealloc(delete_array, sizeof(*delete_array) * (delete_arrays + s_arg[i_arg].n_items - 2));
272 for (i = 2; i < s_arg[i_arg].n_items; i++)
273 delete_array[i - 2 + delete_arrays] = expand_ranges(s_arg[i_arg].list[i]);
274 delete_arrays += s_arg[i_arg].n_items - 2;
275 break;
276 default:
277 SDDS_Bomb("invalid -delete syntax: specify column or parameter keyword");
278 break;
279 }
280 break;
281 case SET_RETAIN:
282 if (s_arg[i_arg].n_items < 3)
283 SDDS_Bomb("invalid -retain syntax");
284 switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
285 case COLUMN_MODE:
286 retain_column = trealloc(retain_column, sizeof(*retain_column) * (retain_columns + s_arg[i_arg].n_items - 2));
287 for (i = 2; i < s_arg[i_arg].n_items; i++)
288 retain_column[i - 2 + retain_columns] = expand_ranges(s_arg[i_arg].list[i]);
289 retain_columns += s_arg[i_arg].n_items - 2;
290 break;
291 case PARAMETER_MODE:
292 retain_parameter = trealloc(retain_parameter, sizeof(*retain_parameter) * (retain_parameters + s_arg[i_arg].n_items - 2));
293 for (i = 2; i < s_arg[i_arg].n_items; i++)
294 retain_parameter[i - 2 + retain_parameters] = expand_ranges(s_arg[i_arg].list[i]);
295 retain_parameters += s_arg[i_arg].n_items - 2;
296 break;
297 case ARRAY_MODE:
298 retain_array = trealloc(retain_array, sizeof(*retain_array) * (retain_arrays + s_arg[i_arg].n_items - 2));
299 for (i = 2; i < s_arg[i_arg].n_items; i++)
300 retain_array[i - 2 + retain_arrays] = expand_ranges(s_arg[i_arg].list[i]);
301 retain_arrays += s_arg[i_arg].n_items - 2;
302 break;
303 default:
304 SDDS_Bomb("invalid -retain syntax: specify column or parameter keyword");
305 break;
306 }
307 break;
308 case SET_SPARSE:
309 if ((s_arg[i_arg].n_items >= 2) && (s_arg[i_arg].n_items <= 3)) {
310 if (sscanf(s_arg[i_arg].list[1], "%ld", &sparse) != 1) {
311 bomb("invalid -sparse syntax", USAGE);
312 }
313 if (sparse <= 0) {
314 bomb("invalid -sparse syntax", USAGE);
315 }
316 if (s_arg[i_arg].n_items == 3) {
317 switch (match_string(s_arg[i_arg].list[2], sparse_mode, SPARSE_MODES, 0)) {
318 case SPARSE_AVERAGE:
319 sparse_statistics = 1;
320 break;
321 case SPARSE_MEDIAN:
322 sparse_statistics = 2;
323 break;
324 case SPARSE_MINIMUM:
325 sparse_statistics = 3;
326 break;
327 case SPARSE_MAXIMUM:
328 sparse_statistics = 4;
329 break;
330 default:
331 SDDS_Bomb("invalid -sparse syntax");
332 break;
333 }
334 }
335 } else {
336 bomb("invalid -sparse syntax", USAGE);
337 }
338 break;
339 case SET_COLLAPSE:
340 collapse = 1;
341 if (append) {
342 SDDS_Bomb("-collapse and -append options cannot be used together");
343 break;
344 }
345 break;
346 default:
347 bomb("unrecognized option", USAGE);
348 break;
349 }
350 } else {
351 inputfile = trealloc(inputfile, sizeof(*inputfile) * (inputfiles + 1));
352 inputfile[inputfiles++] = s_arg[i_arg].list[0];
353 }
354 }
355
356 outputfile = NULL;
357 if (inputfiles > 1) {
358 if (pipeFlags & USE_STDIN)
359 SDDS_Bomb("too many input files with -pipe option");
360 if (!(pipeFlags & USE_STDOUT)) {
361 if (!append) {
362 outputfile = inputfile[--inputfiles];
363 if (fexists(outputfile) && !overwrite)
364 SDDS_Bomb("output file exists already--give -overWrite option to force replacement");
365 }
366 }
367 } else if (inputfiles == 1) {
368 if (pipeFlags & USE_STDIN) {
369 outputfile = inputfile[0];
370 inputfile[0] = NULL;
371 }
372 if (pipeFlags & USE_STDOUT && outputfile)
373 SDDS_Bomb("too many filenames given with -pipe=output");
374 } else {
375 if (!(pipeFlags & USE_STDIN) || !(pipeFlags & USE_STDOUT))
376 SDDS_Bomb("too few filenames given");
377 inputfiles = 1;
378 inputfile = tmalloc(sizeof(*inputfile) * 1);
379 inputfile[0] = outputfile = NULL;
380 }
381
382 for (i = 0; i < inputfiles; i++)
383 if (inputfile[i] && outputfile && strcmp(inputfile[i], outputfile) == 0)
384 SDDS_Bomb("Output file is also an input file.");
385
386 if (append) {
387 if (merge) {
388 int64_t rowsPresent;
389 if (!SDDS_InitializeAppendToPage(&SDDS_output, inputfile[0], 100, &rowsPresent))
390 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
391 } else {
392 if (!SDDS_InitializeAppend(&SDDS_output, inputfile[0]))
393 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
394 }
395 iFile = 1;
396 } else {
397 if (!SDDS_InitializeInput(&SDDS_input, inputfile[0]) ||
398 !SDDS_GetDescription(&SDDS_input, &text, &contents) ||
399 !SDDS_InitializeOutput(&SDDS_output, SDDS_BINARY, 0, text, contents, outputfile))
400 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
401 if (columnMajorOrder != -1)
402 SDDS_output.layout.data_mode.column_major = columnMajorOrder;
403 else
404 SDDS_output.layout.data_mode.column_major = SDDS_input.layout.data_mode.column_major;
405 iFile = 0;
406 }
407
408 for (; iFile < inputfiles; iFile++) {
409 char **name;
410 int32_t names;
411 if (iFile && !SDDS_InitializeInput(&SDDS_input, inputfile[iFile]))
412 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
413 if (!collapse) {
414 if (!(name = SDDS_GetColumnNames(&SDDS_input, &names)))
415 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
416 for (i = 0; i < names; i++) {
417 if (append) {
418 // We need all files to have the same columns for appending
419 if (keep_element(name[i], delete_column, delete_columns, retain_column, retain_columns) &&
420 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0)) {
421 fprintf(stderr, "Error (sddscombine): Problem appending data. Column %s does not exist in first page.\n", name[i]);
422 exit(EXIT_FAILURE);
423 }
424 } else {
425 if (keep_element(name[i], delete_column, delete_columns, retain_column, retain_columns) &&
426 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0 &&
427 !SDDS_TransferColumnDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
428 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
429 }
430 free(name[i]);
431 }
432 free(name);
433 }
434
435 if (!(name = SDDS_GetParameterNames(&SDDS_input, &names)))
436 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
437 for (i = 0; i < names; i++) {
438 if (collapse) {
439 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
440 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0 &&
441 !SDDS_DefineColumnLikeParameter(&SDDS_output, &SDDS_input, name[i], name[i])))
442 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
443 } else {
444 if (append) {
445 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
446 (SDDS_GetParameterIndex(&SDDS_output, name[i]) < 0)) {
447 fprintf(stderr, "Error (sddscombine): Problem appending data. Parameter %s does not exist in first page.\n", name[i]);
448 exit(EXIT_FAILURE);
449 }
450 } else {
451 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
452 (SDDS_GetParameterIndex(&SDDS_output, name[i]) < 0 &&
453 !SDDS_TransferParameterDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
454 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
455 }
456 }
457 free(name[i]);
458 }
459 free(name);
460
461 if (!collapse) {
462 if (!(name = SDDS_GetArrayNames(&SDDS_input, &names)))
463 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
464 for (i = 0; i < names; i++) {
465 if (append) {
466 if (keep_element(name[i], delete_array, delete_arrays, retain_array, retain_arrays) &&
467 (SDDS_GetArrayIndex(&SDDS_output, name[i]) < 0)) {
468 fprintf(stderr, "Error (sddscombine): Problem appending data. Array %s does not exist in first page.\n", name[i]);
469 exit(EXIT_FAILURE);
470 }
471 } else {
472 if (keep_element(name[i], delete_array, delete_arrays, retain_array, retain_arrays) &&
473 (SDDS_GetArrayIndex(&SDDS_output, name[i]) < 0 &&
474 !SDDS_TransferArrayDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
475 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
476 }
477 free(name[i]);
478 }
479 free(name);
480 }
481 if (inputfiles > 1 && !SDDS_Terminate(&SDDS_input))
482 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
483 }
484
485 if (collapse) {
486 if (!(column = SDDS_GetColumnNames(&SDDS_output, &columns))) {
487 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
488 exit(EXIT_FAILURE);
489 }
490 }
491
492 if (collapse) {
493 if (!merge) {
494 if (SDDS_GetColumnIndex(&SDDS_output, "Filename") < 0 &&
495 SDDS_DefineColumn(&SDDS_output, "Filename", NULL, NULL, "Name of file from which this page came", NULL, SDDS_STRING, 0) < 0)
496 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
497 }
498 if (SDDS_GetColumnIndex(&SDDS_output, "NumberCombined") < 0 &&
499 SDDS_DefineColumn(&SDDS_output, "NumberCombined", NULL, NULL, "Number of files combined to make this file", NULL, SDDS_LONG, 0) < 0)
500 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
501 } else {
502 if (!append) {
503 if (!SDDS_DeleteParameterFixedValues(&SDDS_output)) {
504 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
505 exit(EXIT_FAILURE);
506 }
507 if (!merge) {
508 if (SDDS_GetParameterIndex(&SDDS_output, "Filename") < 0 &&
509 SDDS_DefineParameter(&SDDS_output, "Filename", NULL, NULL, "Name of file from which this page came", NULL, SDDS_STRING, NULL) < 0)
510 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
511 }
512 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") < 0 &&
513 SDDS_DefineParameter(&SDDS_output, "NumberCombined", NULL, NULL, "Number of files combined to make this file", NULL, SDDS_LONG, NULL) < 0)
514 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
515 }
516 }
517
518 if (collapse) {
519 if (SDDS_GetColumnIndex(&SDDS_output, "PageNumber") < 0) {
520 if (SDDS_DefineColumn(&SDDS_output, "PageNumber", NULL, NULL, NULL, NULL, SDDS_LONG, 0) < 0) {
521 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
522 exit(EXIT_FAILURE);
523 }
524 setPageNumber = 1;
525 } else {
526 setPageNumber = 0;
527 }
528 }
529 if (!append) {
530 if (!SDDS_WriteLayout(&SDDS_output)) {
531 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
532 exit(EXIT_FAILURE);
533 }
534 }
535 if (collapse) {
536 if (!SDDS_StartPage(&SDDS_output, allocated_rows = ROW_INCREMENT)) {
537 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
538 exit(EXIT_FAILURE);
539 }
540 }
541 nColumns = SDDS_ColumnCount(&SDDS_output);
542 if (append) {
543 iFile = 1;
544 first_data = 0;
545 output_pending = 1;
546 } else {
547 iFile = 0;
548 first_data = 1; /* indicates no pages copied so far */
549 output_pending = 0;
550 }
551
552 page = 0;
553 for (; iFile < inputfiles; iFile++) {
554 if (inputfiles > 1 && !SDDS_InitializeInput(&SDDS_input, inputfile[iFile])) {
555 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
556 exit(EXIT_FAILURE);
557 }
558 first_page = 1;
559 recovered = 0;
560 while (!recovered && (retval = SDDS_ReadPageSparse(&SDDS_input, 0, nColumns ? sparse : INT64_MAX - 1, 0, sparse_statistics)) >= 0) {
561 page++;
562 if (retval == 0) {
563 if (!recover)
564 break;
565 recovered = 1;
566 if (recover == 2 || !SDDS_ReadRecoveryPossible(&SDDS_input))
567 /* user doesn't want this page, or it can't be recovered */
568 break;
569 }
570 if (param) {
571 if (first_page) {
572 if ((param_index = SDDS_GetParameterIndex(&SDDS_input, param)) < 0)
573 SDDS_Bomb("-merge parameter not in input file(s)");
574 if (param_type) {
575 if (param_type != SDDS_GetParameterType(&SDDS_input, param_index))
576 SDDS_Bomb("-merge parameter changes type in subsequent files");
577 } else {
578 param_size = SDDS_GetTypeSize(param_type = SDDS_GetParameterType(&SDDS_input, param_index));
579 this_param = tmalloc(param_size);
580 last_param = tmalloc(param_size);
581 if (!SDDS_GetParameter(&SDDS_input, param, last_param))
582 SDDS_Bomb("error getting value for -merge parameter");
583 }
584 } else {
585 memcpy(last_param, this_param, param_size);
586 }
587 if (!SDDS_GetParameter(&SDDS_input, param, this_param))
588 SDDS_Bomb("error getting value for -merge parameter");
589 }
590#ifdef DEBUG
591 if (param) {
592 fprintf(stderr, "parameter %s = ", param);
593 SDDS_PrintTypedValue(this_param, 0, param_type, NULL, stderr, 0);
594 fprintf(stderr, " now (was ");
595 SDDS_PrintTypedValue(last_param, 0, param_type, NULL, stderr, 0);
596 fprintf(stderr, ")\n");
597 }
598#endif
599 if (collapse) {
600 if (merge && param) {
601 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0 && output_pending) {
602 output_pending = 0;
603 }
604 }
605 if (!merge || (!param && first_data && first_page) || (param && !output_pending)) {
606 if (page > allocated_rows) {
607 if (!SDDS_LengthenTable(&SDDS_output, ROW_INCREMENT)) {
608 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
609 exit(EXIT_FAILURE);
610 }
611 allocated_rows += ROW_INCREMENT;
612 }
613 for (i = 0; i < columns; i++) {
614 if (!SDDS_GetParameterMod(&SDDS_input, &SDDS_output, column[i], buffer)) {
615 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
616 exit(EXIT_FAILURE);
617 }
618 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_REFERENCE, page - 1, column[i], buffer, NULL)) {
619 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
620 exit(EXIT_FAILURE);
621 }
622 }
623 if (!merge) {
624 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "Filename", inputfile[iFile] ? inputfile[iFile] : "stdin", "NumberCombined", inputfiles, NULL)) {
625 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
626 exit(EXIT_FAILURE);
627 }
628 } else {
629 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "NumberCombined", inputfiles, NULL)) {
630 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
631 exit(EXIT_FAILURE);
632 }
633 }
634 if (setPageNumber && !SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "PageNumber", page, NULL)) {
635 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
636 exit(EXIT_FAILURE);
637 }
638 first_data = 0;
639 } else if (merge && param && output_pending) {
640 page--;
641 }
642 } else {
643 if (!merge) {
644 if (!SDDS_ClearPage(&SDDS_output) ||
645 !SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
646 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
647 exit(EXIT_FAILURE);
648 }
649 if (SDDS_GetParameterIndex(&SDDS_output, "Filename") >= 0) {
650 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "Filename", inputfile[iFile] ? inputfile[iFile] : "stdin", NULL)) {
651 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
652 exit(EXIT_FAILURE);
653 }
654 }
655 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
656 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
657 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
658 exit(EXIT_FAILURE);
659 }
660 }
661 if (!SDDS_WritePage(&SDDS_output)) {
662 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
663 exit(EXIT_FAILURE);
664 }
665 } else if (merge && !param) {
666 if (nMerge > 0 && (page - 1) % nMerge == 0 && page != 1) {
667 if (!SDDS_WritePage(&SDDS_output)) {
668 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
669 exit(EXIT_FAILURE);
670 }
671 output_pending = 0;
672 }
673 if ((first_data && first_page) || (nMerge > 0 && (page - 1) % nMerge == 0)) {
674 if (!SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
675 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
676 exit(EXIT_FAILURE);
677 }
678 first_data = 0;
679 } else {
680 if (!SDDS_CopyAdditionalRows(&SDDS_output, &SDDS_input)) {
681 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
682 exit(EXIT_FAILURE);
683 }
684 }
685 } else {
686#ifdef DEBUG
687 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0)
688 fprintf(stderr, "Parameter value has changed\n");
689#endif
690 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0 && output_pending) {
691 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
692 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
693 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
694 exit(EXIT_FAILURE);
695 }
696 }
697 if (!SDDS_WritePage(&SDDS_output)) {
698 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
699 exit(EXIT_FAILURE);
700 }
701 output_pending = 0;
702 }
703 if (!output_pending) {
704 if (!SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
705 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
706 exit(EXIT_FAILURE);
707 }
708 } else {
709 if (!SDDS_CopyAdditionalRows(&SDDS_output, &SDDS_input)) {
710 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
711 exit(EXIT_FAILURE);
712 }
713 }
714 }
715 }
716 if (merge) {
717 output_pending = 1;
718 }
719 first_page = 0;
720 }
721 if (!recovered && (retval == 0 || SDDS_NumberOfErrors() || !SDDS_Terminate(&SDDS_input))) {
722 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
723 exit(EXIT_FAILURE);
724 }
725 }
726 if (!collapse && merge && output_pending) {
727 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
728 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
729 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
730 exit(EXIT_FAILURE);
731 }
732 }
733 if (append) {
734 if (!SDDS_UpdatePage(&SDDS_output, FLUSH_TABLE)) {
735 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
736 exit(EXIT_FAILURE);
737 }
738 } else {
739 if (!SDDS_WritePage(&SDDS_output)) {
740 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
741 exit(EXIT_FAILURE);
742 }
743 }
744 }
745 if (collapse) {
746 if (!SDDS_WritePage(&SDDS_output)) {
747 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
748 exit(EXIT_FAILURE);
749 }
750 if (page == 0) {
751 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
752 exit(EXIT_FAILURE);
753 }
754 }
755 if (!SDDS_Terminate(&SDDS_output)) {
756 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
757 exit(EXIT_FAILURE);
758 }
759
760 return EXIT_SUCCESS;
761}
int32_t SDDS_ReadRecoveryPossible(SDDS_DATASET *SDDS_dataset)
Checks if any data in an SDDS page was recovered after an error was detected.
int32_t SDDS_CopyAdditionalRows(SDDS_DATASET *SDDS_target, SDDS_DATASET *SDDS_source)
Definition SDDS_copy.c:519
int32_t SDDS_CopyPage(SDDS_DATASET *SDDS_target, SDDS_DATASET *SDDS_source)
Definition SDDS_copy.c:578
int32_t SDDS_LengthenTable(SDDS_DATASET *SDDS_dataset, int64_t n_additional_rows)
int32_t SDDS_SetRowValues(SDDS_DATASET *SDDS_dataset, int32_t mode, int64_t row,...)
int32_t SDDS_StartPage(SDDS_DATASET *SDDS_dataset, int64_t expected_n_rows)
int32_t SDDS_SetParameters(SDDS_DATASET *SDDS_dataset, int32_t mode,...)
int32_t SDDS_ClearPage(SDDS_DATASET *SDDS_dataset)
void * SDDS_GetParameter(SDDS_DATASET *SDDS_dataset, char *parameter_name, void *memory)
Retrieves the value of a specified parameter from the current data table of a data set.
int32_t SDDS_GetDescription(SDDS_DATASET *SDDS_dataset, char **text, char **contents)
Retrieves the text and contents descriptions from an SDDS dataset.
int32_t SDDS_ReadPageSparse(SDDS_DATASET *SDDS_dataset, uint32_t mode, int64_t sparse_interval, int64_t sparse_offset, int32_t sparse_statistics)
int32_t SDDS_InitializeInput(SDDS_DATASET *SDDS_dataset, char *filename)
Definition SDDS_input.c:49
int32_t SDDS_Terminate(SDDS_DATASET *SDDS_dataset)
int32_t SDDS_InitializeAppend(SDDS_DATASET *SDDS_dataset, const char *filename)
Initializes the SDDS dataset for appending data by adding a new page to an existing file.
int32_t SDDS_InitializeOutput(SDDS_DATASET *SDDS_dataset, int32_t data_mode, int32_t lines_per_row, const char *description, const char *contents, const char *filename)
Initializes the SDDS output dataset.
int32_t SDDS_InitializeAppendToPage(SDDS_DATASET *SDDS_dataset, const char *filename, int64_t updateInterval, int64_t *rowsPresentReturn)
Initializes the SDDS dataset for appending data to the last page of an existing file.
int32_t SDDS_UpdatePage(SDDS_DATASET *SDDS_dataset, uint32_t mode)
Updates the current page of the SDDS dataset.
int32_t SDDS_WritePage(SDDS_DATASET *SDDS_dataset)
Writes the current data table to the output file.
int32_t SDDS_DefineColumn(SDDS_DATASET *SDDS_dataset, const char *name, const char *symbol, const char *units, const char *description, const char *format_string, int32_t type, int32_t field_length)
Defines a data column within the SDDS dataset.
int32_t SDDS_WriteLayout(SDDS_DATASET *SDDS_dataset)
Writes the SDDS layout header to the output file.
int32_t SDDS_DefineParameter(SDDS_DATASET *SDDS_dataset, const char *name, const char *symbol, const char *units, const char *description, const char *format_string, int32_t type, char *fixed_value)
Defines a data parameter with a fixed string value.
int32_t SDDS_TransferColumnDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a column definition from a source dataset to a target dataset.
int32_t SDDS_TransferArrayDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers an array definition from a source dataset to a target dataset.
int32_t SDDS_DefineColumnLikeParameter(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Defines a column in the target dataset based on a parameter definition from the source dataset.
int32_t SDDS_TransferParameterDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a parameter definition from a source dataset to a target dataset.
int32_t SDDS_GetParameterType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a parameter in the SDDS dataset by its index.
int32_t SDDS_GetArrayIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named array in the SDDS dataset.
char ** SDDS_GetParameterNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all parameters in the SDDS dataset.
int32_t SDDS_GetParameterIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named parameter in the SDDS dataset.
int32_t SDDS_GetColumnIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named column in the SDDS dataset.
int32_t SDDS_ColumnCount(SDDS_DATASET *page)
Retrieves the number of columns in the SDDS dataset.
char ** SDDS_GetColumnNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all columns in the SDDS dataset.
void SDDS_PrintErrors(FILE *fp, int32_t mode)
Prints recorded error messages to a specified file stream.
Definition SDDS_utils.c:432
int32_t SDDS_DeleteParameterFixedValues(SDDS_DATASET *SDDS_dataset)
Deletes fixed values from all parameters in the SDDS dataset.
void SDDS_RegisterProgramName(const char *name)
Registers the executable program name for use in error messages.
Definition SDDS_utils.c:288
int32_t SDDS_NumberOfErrors()
Retrieves the number of errors recorded by SDDS library routines.
Definition SDDS_utils.c:304
int32_t SDDS_GetTypeSize(int32_t type)
Retrieves the size in bytes of a specified SDDS data type.
void SDDS_Bomb(char *message)
Terminates the program after printing an error message and recorded errors.
Definition SDDS_utils.c:342
char ** SDDS_GetArrayNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all arrays in the SDDS dataset.
int32_t SDDS_PrintTypedValue(void *data, int64_t index, int32_t type, char *format, FILE *fp, uint32_t mode)
Prints a data value of a specified type using an optional printf format string.
Definition SDDS_utils.c:59
#define SDDS_STRING
Identifier for the string data type.
Definition SDDStypes.h:85
#define SDDS_LONG
Identifier for the signed 32-bit integer data type.
Definition SDDStypes.h:61
void * trealloc(void *old_ptr, uint64_t size_of_block)
Reallocates a memory block to a new size.
Definition array.c:181
void * tmalloc(uint64_t size_of_block)
Allocates a memory block of the specified size with zero initialization.
Definition array.c:59
void bomb(char *error, char *usage)
Reports error messages to the terminal and aborts the program.
Definition bomb.c:26
long fexists(const char *filename)
Checks if a file exists.
Definition fexists.c:27
long match_string(char *string, char **option, long n_options, long mode)
Matches a given string against an array of option strings based on specified modes.
int scanargs(SCANNED_ARG **scanned, int argc, char **argv)
Definition scanargs.c:36
long processPipeOption(char **item, long items, unsigned long *flags)
Definition scanargs.c:356
long scanItemList(unsigned long *flags, char **item, long *items, unsigned long mode,...)
Scans a list of items and assigns values based on provided keywords and types.
char * expand_ranges(char *template)
Expand range specifiers in a wildcard template into explicit character lists.
Definition wild_match.c:429

◆ SDDS_CompareParameterValues()

long SDDS_CompareParameterValues ( void * param1,
void * param2,
long type )

Definition at line 865 of file sddscombine.c.

865 {
866 double ddiff;
867 int64_t ldiff;
868 char cdiff;
869
870 switch (type) {
871 case SDDS_FLOAT:
872 ddiff = *((float *)param1) - *((float *)param2);
873 return ddiff < 0 ? -1 : ddiff > 0 ? 1 : 0;
874 case SDDS_DOUBLE:
875 ddiff = *((double *)param1) - *((double *)param2);
876 return ddiff < 0 ? -1 : ddiff > 0 ? 1 : 0;
877 case SDDS_LONG64:
878 ldiff = *((int64_t *)param1) - *((int64_t *)param2);
879 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
880 case SDDS_ULONG64:
881 ldiff = *((uint64_t *)param1) - *((uint64_t *)param2);
882 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
883 case SDDS_LONG:
884 ldiff = *((int32_t *)param1) - *((int32_t *)param2);
885 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
886 case SDDS_ULONG:
887 ldiff = *((uint32_t *)param1) - *((uint32_t *)param2);
888 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
889 case SDDS_SHORT:
890 ldiff = *((short *)param1) - *((short *)param2);
891 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
892 case SDDS_USHORT:
893 ldiff = *((unsigned short *)param1) - *((unsigned short *)param2);
894 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
895 case SDDS_CHARACTER:
896 cdiff = (short)*((char *)param1) - (short)*((char *)param2);
897 return cdiff < 0 ? -1 : cdiff > 0 ? 1 : 0;
898 case SDDS_STRING:
899 return strcmp(*(char **)param1, *(char **)param2);
900 default:
901 SDDS_SetError("Problem doing data comparison--invalid data type (SDDS_CompareParameterValues)");
902 SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors | SDDS_VERBOSE_PrintErrors);
903 exit(EXIT_FAILURE);
904 }
905}
void SDDS_SetError(char *error_text)
Records an error message in the SDDS error stack.
Definition SDDS_utils.c:379
#define SDDS_ULONG
Identifier for the unsigned 32-bit integer data type.
Definition SDDStypes.h:67
#define SDDS_FLOAT
Identifier for the float data type.
Definition SDDStypes.h:43
#define SDDS_ULONG64
Identifier for the unsigned 64-bit integer data type.
Definition SDDStypes.h:55
#define SDDS_SHORT
Identifier for the signed short integer data type.
Definition SDDStypes.h:73
#define SDDS_CHARACTER
Identifier for the character data type.
Definition SDDStypes.h:91
#define SDDS_USHORT
Identifier for the unsigned short integer data type.
Definition SDDStypes.h:79
#define SDDS_DOUBLE
Identifier for the double data type.
Definition SDDStypes.h:37
#define SDDS_LONG64
Identifier for the signed 64-bit integer data type.
Definition SDDStypes.h:49

◆ SDDS_GetParameterMod()

void * SDDS_GetParameterMod ( SDDS_DATASET * SDDS_dataset,
SDDS_DATASET * SDDS_output,
char * parameter_name,
void * memory )

Definition at line 763 of file sddscombine.c.

763 {
764 long index, type, size;
765 void *data;
766 float floatdata;
767 double doubledata;
768 uint64_t ulong64data;
769 int64_t long64data;
770 uint32_t ulongdata;
771 int32_t longdata;
772 unsigned short ushortdata;
773 short shortdata;
774 char chardata;
775 char *stringdata;
776 floatdata = 0.0;
777 doubledata = 0.0;
778 ulong64data = 0;
779 long64data = 0;
780 ulongdata = 0;
781 longdata = 0;
782 ushortdata = 0;
783 shortdata = 0;
784 chardata = '\000';
785 stringdata = "";
786
787 if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_GetParameterMod"))
788 return (NULL);
789 if (!parameter_name) {
790 SDDS_SetError("Unable to get parameter value--parameter name pointer is NULL (SDDS_GetParameterMod)");
791 return (NULL);
792 }
793 if ((index = SDDS_GetParameterIndex(SDDS_dataset, parameter_name)) < 0) {
794 if ((index = SDDS_GetColumnIndex(SDDS_output, parameter_name)) < 0) {
795 SDDS_SetError("Unable to get parameter value--parameter name is unrecognized (SDDS_GetParameterMod)");
796 return (NULL);
797 }
798 if (!(type = SDDS_GetColumnType(SDDS_output, index))) {
799 SDDS_SetError("Unable to get parameter value--parameter data type is invalid (SDDS_GetParameterMod)");
800 return (NULL);
801 }
802 size = SDDS_type_size[type - 1];
803 if (memory)
804 data = memory;
805 else if (!(data = SDDS_Malloc(size))) {
806 SDDS_SetError("Unable to get parameter value--parameter data size is invalid (SDDS_GetParameterMod)");
807 return (NULL);
808 }
809 switch (type) {
810 case SDDS_FLOAT:
811 data = memcpy(data, &floatdata, size);
812 break;
813 case SDDS_DOUBLE:
814 data = memcpy(data, &doubledata, size);
815 break;
816 case SDDS_ULONG64:
817 data = memcpy(data, &ulong64data, size);
818 break;
819 case SDDS_LONG64:
820 data = memcpy(data, &long64data, size);
821 break;
822 case SDDS_ULONG:
823 data = memcpy(data, &ulongdata, size);
824 break;
825 case SDDS_LONG:
826 data = memcpy(data, &longdata, size);
827 break;
828 case SDDS_USHORT:
829 data = memcpy(data, &ushortdata, size);
830 break;
831 case SDDS_SHORT:
832 data = memcpy(data, &shortdata, size);
833 break;
834 case SDDS_CHARACTER:
835 data = memcpy(data, &chardata, size);
836 break;
837 case SDDS_STRING:
838 if (!SDDS_CopyString((char **)data, stringdata))
839 break;
840 }
841 } else {
842 if (!(type = SDDS_GetParameterType(SDDS_dataset, index))) {
843 SDDS_SetError("Unable to get parameter value--parameter data type is invalid (SDDS_GetParameterMod)");
844 return (NULL);
845 }
846 if (!SDDS_dataset->parameter || !SDDS_dataset->parameter[index]) {
847 SDDS_SetError("Unable to get parameter value--parameter data array is NULL (SDDS_GetParameterMod)");
848 return (NULL);
849 }
850 size = SDDS_type_size[type - 1];
851 if (memory)
852 data = memory;
853 else if (!(data = SDDS_Malloc(size))) {
854 SDDS_SetError("Unable to get parameter value--parameter data size is invalid (SDDS_GetParameterMod)");
855 return (NULL);
856 }
857 if (type != SDDS_STRING)
858 memcpy(data, SDDS_dataset->parameter[index], size);
859 else if (!SDDS_CopyString((char **)data, *(char **)SDDS_dataset->parameter[index]))
860 return (NULL);
861 }
862 return (data);
863}
int32_t SDDS_type_size[SDDS_NUM_TYPES]
Array of sizes for each supported data type.
Definition SDDS_data.c:62
int32_t SDDS_CheckDataset(SDDS_DATASET *SDDS_dataset, const char *caller)
Validates the SDDS dataset pointer.
Definition SDDS_utils.c:552
void * SDDS_Malloc(size_t size)
Allocates memory of a specified size.
Definition SDDS_utils.c:639
int32_t SDDS_GetColumnType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a column in the SDDS dataset by its index.
int32_t SDDS_CopyString(char **target, const char *source)
Copies a source string to a target string with memory allocation.
Definition SDDS_utils.c:856