SDDS ToolKit Programs and Libraries for C and Python
Loading...
Searching...
No Matches
sddscombine.c
Go to the documentation of this file.
1/**
2 * @file sddscombine.c
3 * @brief Combines multiple SDDS files into a single SDDS file with options for merging, appending, and modifying data.
4 *
5 * @details
6 * This program processes multiple SDDS files, allowing users to merge data, append new pages, or perform transformations
7 * such as deleting or retaining specific columns, parameters, or arrays. It supports various modes for handling sparse data,
8 * recovery of corrupted data, and control over the order of the output dataset.
9 *
10 * @section Usage
11 * ```
12 * sddscombine [<SDDSinputfilelist>] [<SDDSoutputfile>]
13 * [-pipe=[input][,output]]
14 * [-delete={column|parameter|array},<matching-string>[,...]]
15 * [-retain={column|parameter|array},<matching-string>[,...]]
16 * [-sparse=<integer>[,{average|median|minimum|maximum}]]
17 * [-merge[=<parameter-name>|<npages>]]
18 * [-append]
19 * [-overWrite]
20 * [-collapse]
21 * [-recover[=clip]]
22 * [-majorOrder=row|column]
23 * [-xzLevel=<0-9>]
24 * [-threads=<number>]
25 * ```
26 *
27 * @section Options
28 * | Option | Description |
29 * |--------------------------------------|-------------------------------------------------------------------------------------------------|
30 * | `-pipe` | Enable piping for input and/or output. |
31 * | `-delete` | Delete columns, parameters, or arrays matching the specified pattern. |
32 * | `-retain` | Retain only columns, parameters, or arrays matching the specified pattern. |
33 * | `-sparse` | Sample every nth row, optionally performing statistical analysis (`average`, `median`, etc.). |
34 * | `-merge` | Merge pages based on a parameter or number of pages. |
35 * | `-append` | Append data to the first input file. |
36 * | `-overWrite` | Overwrite the output file if it exists. |
37 * | `-collapse` | Collapse the output into a single page, as processed through `sddscollapse`. |
38 * | `-recover` | Recover incomplete or corrupted data, optionally clipping incomplete pages. |
39 * | `-majorOrder` | Specify row-major or column-major order for output. |
40 * | `-xzLevel` | Set LZMA compression level when writing .xz files. |
41 * | `-threads` | Number of input files to read concurrently in ordinary combine mode. |
42 *
43 * @subsection Incompatibilities
44 * - `-collapse` is incompatible with:
45 * - `-append`
46 * - Only one of the following may be specified:
47 * - `-delete`
48 * - `-retain`
49 * - For `-merge`:
50 * - `<parameter-name>` must exist in all input files if specified.
51 *
52 * @copyright
53 * - (c) 2002 The University of Chicago, as Operator of Argonne National Laboratory.
54 * - (c) 2002 The Regents of the University of California, as Operator of Los Alamos National Laboratory.
55 *
56 * @license
57 * This file is distributed under the terms of the Software License Agreement
58 * found in the file LICENSE included with this distribution.
59 *
60 * @authors
61 * M. Borland, C. Saunders, R. Soliday, H. Shang
62 */
63
64#include "mdb.h"
65#include "scan.h"
66#include "SDDS.h"
67
68/* Enumeration for option types */
69enum option_type {
70 SET_MERGE,
71 SET_OVERWRITE,
72 SET_PIPE,
73 SET_DELETE,
74 SET_RETAIN,
75 SET_SPARSE,
76 SET_COLLAPSE,
77 SET_RECOVER,
78 SET_MAJOR_ORDER,
79 SET_APPEND,
80 SET_XZLEVEL,
81 SET_THREADS,
82 N_OPTIONS
83};
84
85static char *option[N_OPTIONS] = {
86 "merge",
87 "overwrite",
88 "pipe",
89 "delete",
90 "retain",
91 "sparse",
92 "collapse",
93 "recover",
94 "majorOrder",
95 "append",
96 "xzlevel",
97 "threads"
98};
99
100char *USAGE =
101 "\n"
102 " sddscombine [<SDDSinputfilelist>] [<SDDSoutputfile>]\n"
103 " [-pipe=[input][,output]]\n"
104 " [-delete={column|parameter|array},<matching-string>[,...]]\n"
105 " [-retain={column|parameter|array},<matching-string>[,...]]\n"
106 " [-sparse=<integer>[,{average|median|minimum|maximum}]]\n"
107 " [-merge[=<parameter-name>|<npages>]]\n"
108 " [-append]\n"
109 " [-overWrite]\n"
110 " [-collapse]\n"
111 " [-recover[=clip]]\n"
112 " [-majorOrder=row|column]\n"
113 " [-xzLevel=<0-9>]\n"
114 " [-threads=<number>]\n\n"
115 "Options:\n"
116 " -pipe=input,output Enable piping for input and/or output.\n"
117 " -delete=type,pattern Delete columns, parameters, or arrays matching the pattern.\n"
118 " -retain=type,pattern Retain only columns, parameters, or arrays matching the pattern.\n"
119 " -sparse=<n>,mode Sample every nth row with optional mode (average, median, minimum, maximum).\n"
120 " -merge=param|npages Merge pages based on a parameter or number of pages.\n"
121 " -append Append data to the first input file.\n"
122 " -overWrite Overwrite the output file if it exists.\n"
123 " -collapse Collapse the output as if processed through sddscollapse.\n"
124 " -recover=clip Recover incomplete/corrupted data, optionally clipping incomplete pages.\n"
125 " -majorOrder=row|column Specify data write order: row-major or column-major.\n"
126 " -xzLevel=<0-9> Set LZMA compression level when writing .xz files.\n\n"
127 " -threads=<number> Read this many input files concurrently in ordinary combine mode. The default is 1.\n\n"
128 "Description:\n"
129 " sddscombine combines data from a series of SDDS files into a single SDDS file, usually with one page for each page in each file. "
130 "Data is added from files in the order that they are listed on the command line. A new parameter ('Filename') is added to show the source of each page.\n\n"
131 "Program by Michael Borland. (" __DATE__ " " __TIME__ ", SVN revision: " SVN_VERSION ")\n";
132
133long SDDS_CompareParameterValues(void *param1, void *param2, long type);
134long keep_element(char *name, char **delete, long deletions, char **retain, long retentions);
135void *SDDS_GetParameterMod(SDDS_DATASET *SDDS_dataset, SDDS_DATASET *SDDS_output, char *parameter_name, void *memory);
136static void WriteStandardPage(SDDS_DATASET *SDDS_output, SDDS_DATASET *SDDS_input, char *inputfile, long inputfiles);
137static void CombineStandardInputThreaded(SDDS_DATASET *SDDS_output, char **inputfile, long inputfiles, long sparse, int32_t sparse_statistics, long nColumns, int threads);
138
139#define COLUMN_MODE 0
140#define PARAMETER_MODE 1
141#define ARRAY_MODE 2
142#define MODES 3
143char *mode_name[MODES] = {
144 "column",
145 "parameter",
146 "array",
147};
148
149#define SPARSE_AVERAGE 0
150#define SPARSE_MEDIAN 1
151#define SPARSE_MINIMUM 2
152#define SPARSE_MAXIMUM 3
153#define SPARSE_MODES 4
154char *sparse_mode[SPARSE_MODES] = {
155 "average",
156 "median",
157 "minimum",
158 "maximum"
159};
160
161#define ROW_INCREMENT 100
162
163int main(int argc, char **argv) {
164 SDDS_DATASET SDDS_input, SDDS_output;
165 char **inputfile, *outputfile;
166 long inputfiles, i, i_arg, retval = 0, first_page;
167 long iFile, first_data, sparse, setPageNumber;
168 int32_t sparse_statistics = 0;
169 long merge, nMerge, overwrite, collapse, page, append;
170 int64_t allocated_rows;
171 int32_t columns;
172 SCANNED_ARG *s_arg;
173 long buffer[16];
174 char *param, *last_param, *this_param, *text, *contents, **column;
175 long param_index, param_type, param_size, output_pending;
176 unsigned long pipeFlags, majorOrderFlag;
177 long xzLevel = SDDS_GetLZMACompressionLevel();
178
179 char **retain_column, **delete_column;
180 long retain_columns, delete_columns;
181 char **retain_parameter, **delete_parameter;
182 long retain_parameters, delete_parameters;
183 char **retain_array, **delete_array;
184 long retain_arrays, delete_arrays;
185 long nColumns, recover, recovered;
186 short columnMajorOrder = -1;
187 int threads = 1;
188
189 SDDS_RegisterProgramName(argv[0]);
190 argc = scanargs(&s_arg, argc, argv);
191 if (argc < 3)
192 bomb(NULL, USAGE);
193
194 setPageNumber = allocated_rows = param_type = param_size = 0;
195 last_param = this_param = NULL;
196 column = NULL;
197 inputfile = NULL;
198 outputfile = param = NULL;
199 inputfiles = merge = overwrite = collapse = append = nMerge = 0;
200 pipeFlags = 0;
201 sparse = 1;
202 recover = 0;
203
204 retain_column = delete_column = NULL;
205 retain_columns = delete_columns = 0;
206 retain_parameter = delete_parameter = NULL;
207 retain_parameters = delete_parameters = 0;
208 retain_array = delete_array = NULL;
209 retain_arrays = delete_arrays = 0;
210
211 SDDS_RegisterProgramName(argv[0]);
212 argc = scanargs(&s_arg, argc, argv);
213 if (argc < 3)
214 bomb(NULL, USAGE);
215
216 for (i_arg = 1; i_arg < argc; i_arg++) {
217 if (s_arg[i_arg].arg_type == OPTION) {
218 switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
219 case SET_MAJOR_ORDER:
220 majorOrderFlag = 0;
221 s_arg[i_arg].n_items -= 1;
222 if (s_arg[i_arg].n_items > 0 &&
223 (!scanItemList(&majorOrderFlag, s_arg[i_arg].list + 1, &s_arg[i_arg].n_items, 0,
224 "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
225 "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL)))
226 SDDS_Bomb("invalid -majorOrder syntax/values");
227 if (majorOrderFlag & SDDS_COLUMN_MAJOR_ORDER)
228 columnMajorOrder = 1;
229 else if (majorOrderFlag & SDDS_ROW_MAJOR_ORDER)
230 columnMajorOrder = 0;
231 break;
232 case SET_XZLEVEL:
233 if (s_arg[i_arg].n_items != 2 || sscanf(s_arg[i_arg].list[1], "%ld", &xzLevel) != 1 || xzLevel < 0 || xzLevel > 9)
234 SDDS_Bomb("invalid -xzLevel syntax");
235 SDDS_SetLZMACompressionLevel(xzLevel);
236 break;
237 case SET_THREADS:
238 if (s_arg[i_arg].n_items != 2 ||
239 sscanf(s_arg[i_arg].list[1], "%d", &threads) != 1 ||
240 threads < 1)
241 SDDS_Bomb("invalid -threads syntax");
242 break;
243 case SET_MERGE:
244 if (s_arg[i_arg].n_items > 2)
245 bomb("invalid -merge syntax", USAGE);
246 merge = 1;
247 param = NULL;
248 nMerge = -1;
249 if (s_arg[i_arg].n_items == 2) {
250 if (isdigit(s_arg[i_arg].list[1][0])) {
251 if (!sscanf(s_arg[i_arg].list[1], "%ld", &nMerge))
252 bomb("invalid -merge syntax (could not scan number of pages)", USAGE);
253 } else
254 param = s_arg[i_arg].list[1];
255 }
256 break;
257 case SET_APPEND:
258 if (s_arg[i_arg].n_items > 1)
259 bomb("invalid -append syntax", USAGE);
260 append = 1;
261 if (collapse) {
262 SDDS_Bomb("-collapse and -append options cannot be used together");
263 break;
264 }
265 break;
266 case SET_OVERWRITE:
267 overwrite = 1;
268 break;
269 case SET_PIPE:
270 if (!processPipeOption(s_arg[i_arg].list + 1, s_arg[i_arg].n_items - 1, &pipeFlags))
271 SDDS_Bomb("invalid -pipe syntax");
272 break;
273 case SET_RECOVER:
274 recover = 1;
275 if (s_arg[i_arg].n_items != 1) {
276 recover = 2;
277 if (s_arg[i_arg].n_items > 2 || strncmp(s_arg[i_arg].list[1], "clip", strlen(s_arg[i_arg].list[1])) != 0)
278 SDDS_Bomb("invalid -recover syntax");
279 }
280 break;
281 case SET_DELETE:
282 if (s_arg[i_arg].n_items < 3)
283 SDDS_Bomb("invalid -delete syntax");
284 switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
285 case COLUMN_MODE:
286 delete_column = trealloc(delete_column, sizeof(*delete_column) * (delete_columns + s_arg[i_arg].n_items - 2));
287 for (i = 2; i < s_arg[i_arg].n_items; i++)
288 delete_column[i - 2 + delete_columns] = expand_ranges(s_arg[i_arg].list[i]);
289 delete_columns += s_arg[i_arg].n_items - 2;
290 break;
291 case PARAMETER_MODE:
292 delete_parameter = trealloc(delete_parameter, sizeof(*delete_parameter) * (delete_parameters + s_arg[i_arg].n_items - 2));
293 for (i = 2; i < s_arg[i_arg].n_items; i++)
294 delete_parameter[i - 2 + delete_parameters] = expand_ranges(s_arg[i_arg].list[i]);
295 delete_parameters += s_arg[i_arg].n_items - 2;
296 break;
297 case ARRAY_MODE:
298 delete_array = trealloc(delete_array, sizeof(*delete_array) * (delete_arrays + s_arg[i_arg].n_items - 2));
299 for (i = 2; i < s_arg[i_arg].n_items; i++)
300 delete_array[i - 2 + delete_arrays] = expand_ranges(s_arg[i_arg].list[i]);
301 delete_arrays += s_arg[i_arg].n_items - 2;
302 break;
303 default:
304 SDDS_Bomb("invalid -delete syntax: specify column or parameter keyword");
305 break;
306 }
307 break;
308 case SET_RETAIN:
309 if (s_arg[i_arg].n_items < 3)
310 SDDS_Bomb("invalid -retain syntax");
311 switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
312 case COLUMN_MODE:
313 retain_column = trealloc(retain_column, sizeof(*retain_column) * (retain_columns + s_arg[i_arg].n_items - 2));
314 for (i = 2; i < s_arg[i_arg].n_items; i++)
315 retain_column[i - 2 + retain_columns] = expand_ranges(s_arg[i_arg].list[i]);
316 retain_columns += s_arg[i_arg].n_items - 2;
317 break;
318 case PARAMETER_MODE:
319 retain_parameter = trealloc(retain_parameter, sizeof(*retain_parameter) * (retain_parameters + s_arg[i_arg].n_items - 2));
320 for (i = 2; i < s_arg[i_arg].n_items; i++)
321 retain_parameter[i - 2 + retain_parameters] = expand_ranges(s_arg[i_arg].list[i]);
322 retain_parameters += s_arg[i_arg].n_items - 2;
323 break;
324 case ARRAY_MODE:
325 retain_array = trealloc(retain_array, sizeof(*retain_array) * (retain_arrays + s_arg[i_arg].n_items - 2));
326 for (i = 2; i < s_arg[i_arg].n_items; i++)
327 retain_array[i - 2 + retain_arrays] = expand_ranges(s_arg[i_arg].list[i]);
328 retain_arrays += s_arg[i_arg].n_items - 2;
329 break;
330 default:
331 SDDS_Bomb("invalid -retain syntax: specify column or parameter keyword");
332 break;
333 }
334 break;
335 case SET_SPARSE:
336 if ((s_arg[i_arg].n_items >= 2) && (s_arg[i_arg].n_items <= 3)) {
337 if (sscanf(s_arg[i_arg].list[1], "%ld", &sparse) != 1) {
338 bomb("invalid -sparse syntax", USAGE);
339 }
340 if (sparse <= 0) {
341 bomb("invalid -sparse syntax", USAGE);
342 }
343 if (s_arg[i_arg].n_items == 3) {
344 switch (match_string(s_arg[i_arg].list[2], sparse_mode, SPARSE_MODES, 0)) {
345 case SPARSE_AVERAGE:
346 sparse_statistics = 1;
347 break;
348 case SPARSE_MEDIAN:
349 sparse_statistics = 2;
350 break;
351 case SPARSE_MINIMUM:
352 sparse_statistics = 3;
353 break;
354 case SPARSE_MAXIMUM:
355 sparse_statistics = 4;
356 break;
357 default:
358 SDDS_Bomb("invalid -sparse syntax");
359 break;
360 }
361 }
362 } else {
363 bomb("invalid -sparse syntax", USAGE);
364 }
365 break;
366 case SET_COLLAPSE:
367 collapse = 1;
368 if (append) {
369 SDDS_Bomb("-collapse and -append options cannot be used together");
370 break;
371 }
372 break;
373 default:
374 bomb("unrecognized option", USAGE);
375 break;
376 }
377 } else {
378 inputfile = trealloc(inputfile, sizeof(*inputfile) * (inputfiles + 1));
379 inputfile[inputfiles++] = s_arg[i_arg].list[0];
380 }
381 }
382
383 outputfile = NULL;
384 if (inputfiles > 1) {
385 if (pipeFlags & USE_STDIN)
386 SDDS_Bomb("too many input files with -pipe option");
387 if (!(pipeFlags & USE_STDOUT)) {
388 if (!append) {
389 outputfile = inputfile[--inputfiles];
390 if (fexists(outputfile) && !overwrite)
391 SDDS_Bomb("output file exists already--give -overWrite option to force replacement");
392 }
393 }
394 } else if (inputfiles == 1) {
395 if (pipeFlags & USE_STDIN) {
396 outputfile = inputfile[0];
397 inputfile[0] = NULL;
398 }
399 if (pipeFlags & USE_STDOUT && outputfile)
400 SDDS_Bomb("too many filenames given with -pipe=output");
401 } else {
402 if (!(pipeFlags & USE_STDIN) || !(pipeFlags & USE_STDOUT))
403 SDDS_Bomb("too few filenames given");
404 inputfiles = 1;
405 inputfile = tmalloc(sizeof(*inputfile) * 1);
406 inputfile[0] = outputfile = NULL;
407 }
408
409 for (i = 0; i < inputfiles; i++)
410 if (inputfile[i] && outputfile && strcmp(inputfile[i], outputfile) == 0)
411 SDDS_Bomb("Output file is also an input file.");
412
413 if (append) {
414 if (merge) {
415 int64_t rowsPresent;
416 if (!SDDS_InitializeAppendToPage(&SDDS_output, inputfile[0], 100, &rowsPresent))
417 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
418 } else {
419 if (!SDDS_InitializeAppend(&SDDS_output, inputfile[0]))
420 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
421 }
422 iFile = 1;
423 } else {
424 if (!SDDS_InitializeInput(&SDDS_input, inputfile[0]) ||
425 !SDDS_GetDescription(&SDDS_input, &text, &contents) ||
426 !SDDS_InitializeOutput(&SDDS_output, SDDS_BINARY, 0, text, contents, outputfile))
427 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
428 if (columnMajorOrder != -1)
429 SDDS_output.layout.data_mode.column_major = columnMajorOrder;
430 else
431 SDDS_output.layout.data_mode.column_major = SDDS_input.layout.data_mode.column_major;
432 iFile = 0;
433 }
434
435 for (; iFile < inputfiles; iFile++) {
436 char **name;
437 int32_t names;
438 if (iFile && !SDDS_InitializeInput(&SDDS_input, inputfile[iFile]))
439 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
440 if (!collapse) {
441 if (!(name = SDDS_GetColumnNames(&SDDS_input, &names)))
442 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
443 for (i = 0; i < names; i++) {
444 if (append) {
445 // We need all files to have the same columns for appending
446 if (keep_element(name[i], delete_column, delete_columns, retain_column, retain_columns) &&
447 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0)) {
448 fprintf(stderr, "Error (sddscombine): Problem appending data. Column %s does not exist in first page.\n", name[i]);
449 exit(EXIT_FAILURE);
450 }
451 } else {
452 if (keep_element(name[i], delete_column, delete_columns, retain_column, retain_columns) &&
453 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0 &&
454 !SDDS_TransferColumnDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
455 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
456 }
457 free(name[i]);
458 }
459 free(name);
460 }
461
462 if (!(name = SDDS_GetParameterNames(&SDDS_input, &names)))
463 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
464 for (i = 0; i < names; i++) {
465 if (collapse) {
466 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
467 (SDDS_GetColumnIndex(&SDDS_output, name[i]) < 0 &&
468 !SDDS_DefineColumnLikeParameter(&SDDS_output, &SDDS_input, name[i], name[i])))
469 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
470 } else {
471 if (append) {
472 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
473 (SDDS_GetParameterIndex(&SDDS_output, name[i]) < 0)) {
474 fprintf(stderr, "Error (sddscombine): Problem appending data. Parameter %s does not exist in first page.\n", name[i]);
475 exit(EXIT_FAILURE);
476 }
477 } else {
478 if (keep_element(name[i], delete_parameter, delete_parameters, retain_parameter, retain_parameters) &&
479 (SDDS_GetParameterIndex(&SDDS_output, name[i]) < 0 &&
480 !SDDS_TransferParameterDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
481 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
482 }
483 }
484 free(name[i]);
485 }
486 free(name);
487
488 if (!collapse) {
489 if (!(name = SDDS_GetArrayNames(&SDDS_input, &names)))
490 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
491 for (i = 0; i < names; i++) {
492 if (append) {
493 if (keep_element(name[i], delete_array, delete_arrays, retain_array, retain_arrays) &&
494 (SDDS_GetArrayIndex(&SDDS_output, name[i]) < 0)) {
495 fprintf(stderr, "Error (sddscombine): Problem appending data. Array %s does not exist in first page.\n", name[i]);
496 exit(EXIT_FAILURE);
497 }
498 } else {
499 if (keep_element(name[i], delete_array, delete_arrays, retain_array, retain_arrays) &&
500 (SDDS_GetArrayIndex(&SDDS_output, name[i]) < 0 &&
501 !SDDS_TransferArrayDefinition(&SDDS_output, &SDDS_input, name[i], name[i])))
502 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
503 }
504 free(name[i]);
505 }
506 free(name);
507 }
508 if (inputfiles > 1 && !SDDS_Terminate(&SDDS_input))
509 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
510 }
511
512 if (collapse) {
513 if (!(column = SDDS_GetColumnNames(&SDDS_output, &columns))) {
514 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
515 exit(EXIT_FAILURE);
516 }
517 }
518
519 if (collapse) {
520 if (!merge) {
521 if (SDDS_GetColumnIndex(&SDDS_output, "Filename") < 0 &&
522 SDDS_DefineColumn(&SDDS_output, "Filename", NULL, NULL, "Name of file from which this page came", NULL, SDDS_STRING, 0) < 0)
523 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
524 }
525 if (SDDS_GetColumnIndex(&SDDS_output, "NumberCombined") < 0 &&
526 SDDS_DefineColumn(&SDDS_output, "NumberCombined", NULL, NULL, "Number of files combined to make this file", NULL, SDDS_LONG, 0) < 0)
527 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
528 } else {
529 if (!append) {
530 if (!SDDS_DeleteParameterFixedValues(&SDDS_output)) {
531 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
532 exit(EXIT_FAILURE);
533 }
534 if (!merge) {
535 if (SDDS_GetParameterIndex(&SDDS_output, "Filename") < 0 &&
536 SDDS_DefineParameter(&SDDS_output, "Filename", NULL, NULL, "Name of file from which this page came", NULL, SDDS_STRING, NULL) < 0)
537 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
538 }
539 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") < 0 &&
540 SDDS_DefineParameter(&SDDS_output, "NumberCombined", NULL, NULL, "Number of files combined to make this file", NULL, SDDS_LONG, NULL) < 0)
541 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors | SDDS_EXIT_PrintErrors);
542 }
543 }
544
545 if (collapse) {
546 if (SDDS_GetColumnIndex(&SDDS_output, "PageNumber") < 0) {
547 if (SDDS_DefineColumn(&SDDS_output, "PageNumber", NULL, NULL, NULL, NULL, SDDS_LONG, 0) < 0) {
548 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
549 exit(EXIT_FAILURE);
550 }
551 setPageNumber = 1;
552 } else {
553 setPageNumber = 0;
554 }
555 }
556 if (!append) {
557 if (!SDDS_WriteLayout(&SDDS_output)) {
558 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
559 exit(EXIT_FAILURE);
560 }
561 }
562 if (collapse) {
563 if (!SDDS_StartPage(&SDDS_output, allocated_rows = ROW_INCREMENT)) {
564 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
565 exit(EXIT_FAILURE);
566 }
567 }
568 nColumns = SDDS_ColumnCount(&SDDS_output);
569 if (threads > inputfiles)
570 threads = inputfiles;
571 if (threads > 1 && inputfiles > 1 && !append && !merge && !collapse && !recover) {
572 CombineStandardInputThreaded(&SDDS_output, inputfile, inputfiles, sparse, sparse_statistics, nColumns, threads);
573 if (!SDDS_Terminate(&SDDS_output)) {
574 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
575 exit(EXIT_FAILURE);
576 }
577 return EXIT_SUCCESS;
578 }
579 if (append) {
580 iFile = 1;
581 first_data = 0;
582 output_pending = 1;
583 } else {
584 iFile = 0;
585 first_data = 1; /* indicates no pages copied so far */
586 output_pending = 0;
587 }
588
589 page = 0;
590 for (; iFile < inputfiles; iFile++) {
591 if (inputfiles > 1 && !SDDS_InitializeInput(&SDDS_input, inputfile[iFile])) {
592 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
593 exit(EXIT_FAILURE);
594 }
595 first_page = 1;
596 recovered = 0;
597 while (!recovered && (retval = SDDS_ReadPageSparse(&SDDS_input, 0, nColumns ? sparse : INT64_MAX - 1, 0, sparse_statistics)) >= 0) {
598 page++;
599 if (retval == 0) {
600 if (!recover)
601 break;
602 recovered = 1;
603 if (recover == 2 || !SDDS_ReadRecoveryPossible(&SDDS_input))
604 /* user doesn't want this page, or it can't be recovered */
605 break;
606 }
607 if (param) {
608 if (first_page) {
609 if ((param_index = SDDS_GetParameterIndex(&SDDS_input, param)) < 0)
610 SDDS_Bomb("-merge parameter not in input file(s)");
611 if (param_type) {
612 if (param_type != SDDS_GetParameterType(&SDDS_input, param_index))
613 SDDS_Bomb("-merge parameter changes type in subsequent files");
614 } else {
615 param_size = SDDS_GetTypeSize(param_type = SDDS_GetParameterType(&SDDS_input, param_index));
616 this_param = tmalloc(param_size);
617 last_param = tmalloc(param_size);
618 if (!SDDS_GetParameter(&SDDS_input, param, last_param))
619 SDDS_Bomb("error getting value for -merge parameter");
620 }
621 } else {
622 memcpy(last_param, this_param, param_size);
623 }
624 if (!SDDS_GetParameter(&SDDS_input, param, this_param))
625 SDDS_Bomb("error getting value for -merge parameter");
626 }
627#ifdef DEBUG
628 if (param) {
629 fprintf(stderr, "parameter %s = ", param);
630 SDDS_PrintTypedValue(this_param, 0, param_type, NULL, stderr, 0);
631 fprintf(stderr, " now (was ");
632 SDDS_PrintTypedValue(last_param, 0, param_type, NULL, stderr, 0);
633 fprintf(stderr, ")\n");
634 }
635#endif
636 if (collapse) {
637 if (merge && param) {
638 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0 && output_pending) {
639 output_pending = 0;
640 }
641 }
642 if (!merge || (!param && first_data && first_page) || (param && !output_pending)) {
643 if (page > allocated_rows) {
644 if (!SDDS_LengthenTable(&SDDS_output, ROW_INCREMENT)) {
645 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
646 exit(EXIT_FAILURE);
647 }
648 allocated_rows += ROW_INCREMENT;
649 }
650 for (i = 0; i < columns; i++) {
651 if (!SDDS_GetParameterMod(&SDDS_input, &SDDS_output, column[i], buffer)) {
652 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
653 exit(EXIT_FAILURE);
654 }
655 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_REFERENCE, page - 1, column[i], buffer, NULL)) {
656 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
657 exit(EXIT_FAILURE);
658 }
659 }
660 if (!merge) {
661 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "Filename", inputfile[iFile] ? inputfile[iFile] : "stdin", "NumberCombined", inputfiles, NULL)) {
662 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
663 exit(EXIT_FAILURE);
664 }
665 } else {
666 if (!SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "NumberCombined", inputfiles, NULL)) {
667 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
668 exit(EXIT_FAILURE);
669 }
670 }
671 if (setPageNumber && !SDDS_SetRowValues(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, page - 1, "PageNumber", page, NULL)) {
672 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
673 exit(EXIT_FAILURE);
674 }
675 first_data = 0;
676 } else if (merge && param && output_pending) {
677 page--;
678 }
679 } else {
680 if (!merge) {
681 if (!SDDS_ClearPage(&SDDS_output) ||
682 !SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
683 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
684 exit(EXIT_FAILURE);
685 }
686 if (SDDS_GetParameterIndex(&SDDS_output, "Filename") >= 0) {
687 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "Filename", inputfile[iFile] ? inputfile[iFile] : "stdin", NULL)) {
688 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
689 exit(EXIT_FAILURE);
690 }
691 }
692 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
693 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
694 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
695 exit(EXIT_FAILURE);
696 }
697 }
698 if (!SDDS_WritePage(&SDDS_output)) {
699 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
700 exit(EXIT_FAILURE);
701 }
702 } else if (merge && !param) {
703 if (nMerge > 0 && (page - 1) % nMerge == 0 && page != 1) {
704 if (!SDDS_WritePage(&SDDS_output)) {
705 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
706 exit(EXIT_FAILURE);
707 }
708 output_pending = 0;
709 }
710 if ((first_data && first_page) || (nMerge > 0 && (page - 1) % nMerge == 0)) {
711 if (!SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
712 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
713 exit(EXIT_FAILURE);
714 }
715 first_data = 0;
716 } else {
717 if (!SDDS_CopyAdditionalRows(&SDDS_output, &SDDS_input)) {
718 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
719 exit(EXIT_FAILURE);
720 }
721 }
722 } else {
723#ifdef DEBUG
724 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0)
725 fprintf(stderr, "Parameter value has changed\n");
726#endif
727 if (SDDS_CompareParameterValues(this_param, last_param, param_type) != 0 && output_pending) {
728 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
729 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
730 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
731 exit(EXIT_FAILURE);
732 }
733 }
734 if (!SDDS_WritePage(&SDDS_output)) {
735 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
736 exit(EXIT_FAILURE);
737 }
738 output_pending = 0;
739 }
740 if (!output_pending) {
741 if (!SDDS_CopyPage(&SDDS_output, &SDDS_input)) {
742 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
743 exit(EXIT_FAILURE);
744 }
745 } else {
746 if (!SDDS_CopyAdditionalRows(&SDDS_output, &SDDS_input)) {
747 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
748 exit(EXIT_FAILURE);
749 }
750 }
751 }
752 }
753 if (merge) {
754 output_pending = 1;
755 }
756 first_page = 0;
757 }
758 if (!recovered && (retval == 0 || SDDS_NumberOfErrors() || !SDDS_Terminate(&SDDS_input))) {
759 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
760 exit(EXIT_FAILURE);
761 }
762 }
763 if (!collapse && merge && output_pending) {
764 if (SDDS_GetParameterIndex(&SDDS_output, "NumberCombined") >= 0) {
765 if (!SDDS_SetParameters(&SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
766 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
767 exit(EXIT_FAILURE);
768 }
769 }
770 if (append) {
771 if (!SDDS_UpdatePage(&SDDS_output, FLUSH_TABLE)) {
772 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
773 exit(EXIT_FAILURE);
774 }
775 } else {
776 if (!SDDS_WritePage(&SDDS_output)) {
777 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
778 exit(EXIT_FAILURE);
779 }
780 }
781 }
782 if (collapse) {
783 if (!SDDS_WritePage(&SDDS_output)) {
784 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
785 exit(EXIT_FAILURE);
786 }
787 if (page == 0) {
788 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
789 exit(EXIT_FAILURE);
790 }
791 }
792 if (!SDDS_Terminate(&SDDS_output)) {
793 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
794 exit(EXIT_FAILURE);
795 }
796
797 return EXIT_SUCCESS;
798}
799
800static void WriteStandardPage(SDDS_DATASET *SDDS_output, SDDS_DATASET *SDDS_input, char *inputfile, long inputfiles) {
801 if (!SDDS_ClearPage(SDDS_output) ||
802 !SDDS_CopyPage(SDDS_output, SDDS_input)) {
803 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
804 exit(EXIT_FAILURE);
805 }
806 if (SDDS_GetParameterIndex(SDDS_output, "Filename") >= 0) {
807 if (!SDDS_SetParameters(SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "Filename", inputfile ? inputfile : "stdin", NULL)) {
808 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
809 exit(EXIT_FAILURE);
810 }
811 }
812 if (SDDS_GetParameterIndex(SDDS_output, "NumberCombined") >= 0) {
813 if (!SDDS_SetParameters(SDDS_output, SDDS_SET_BY_NAME | SDDS_PASS_BY_VALUE, "NumberCombined", inputfiles, NULL)) {
814 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
815 exit(EXIT_FAILURE);
816 }
817 }
818 if (!SDDS_WritePage(SDDS_output)) {
819 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
820 exit(EXIT_FAILURE);
821 }
822}
823
824static void CombineStandardInputThreaded(SDDS_DATASET *SDDS_output, char **inputfile, long inputfiles, long sparse, int32_t sparse_statistics, long nColumns, int threads) {
825 long batchStart;
826 int activeThreads;
827 int64_t sparseInterval;
828
829 if (threads < 1)
830 threads = 1;
831 if (threads > inputfiles)
832 threads = inputfiles;
833 sparseInterval = nColumns ? sparse : INT64_MAX - 1;
834
835 for (batchStart = 0; batchStart < inputfiles; batchStart += threads) {
836 SDDS_DATASET *SDDS_input;
837 int *initialized;
838 int32_t *readCode;
839 char (*errorMessage)[1024];
840 int batchSize, slot;
841
842 batchSize = threads;
843 if (batchStart + batchSize > inputfiles)
844 batchSize = inputfiles - batchStart;
845 activeThreads = batchSize;
846 SDDS_input = calloc(batchSize, sizeof(*SDDS_input));
847 initialized = calloc(batchSize, sizeof(*initialized));
848 readCode = calloc(batchSize, sizeof(*readCode));
849 errorMessage = calloc(batchSize, sizeof(*errorMessage));
850 if (!SDDS_input || !initialized || !readCode || !errorMessage)
851 SDDS_Bomb("memory allocation failure");
852
853#pragma omp parallel for if (activeThreads > 1) num_threads(activeThreads) schedule(dynamic)
854 for (slot = 0; slot < batchSize; slot++) {
855 if (SDDS_InitializeInput(&SDDS_input[slot], inputfile[batchStart + slot])) {
856 initialized[slot] = 1;
857 readCode[slot] = SDDS_ReadPageSparse(&SDDS_input[slot], 0, sparseInterval, 0, sparse_statistics);
858 if (readCode[slot] == 0)
859 snprintf(errorMessage[slot], sizeof(*errorMessage), "Error: unable to read '%s'.", inputfile[batchStart + slot]);
860 } else {
861 initialized[slot] = 0;
862 readCode[slot] = 0;
863 snprintf(errorMessage[slot], sizeof(*errorMessage), "Error: unable to open '%s'.", inputfile[batchStart + slot]);
864 }
865 }
866
867 for (slot = 0; slot < batchSize; slot++) {
868 if (!initialized[slot] || readCode[slot] == 0) {
869 fprintf(stderr, "%s\n", errorMessage[slot]);
870 exit(EXIT_FAILURE);
871 }
872 while (readCode[slot] > 0) {
873 WriteStandardPage(SDDS_output, &SDDS_input[slot], inputfile[batchStart + slot], inputfiles);
874 readCode[slot] = SDDS_ReadPageSparse(&SDDS_input[slot], 0, sparseInterval, 0, sparse_statistics);
875 }
876 if (readCode[slot] == 0 || !SDDS_Terminate(&SDDS_input[slot])) {
877 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
878 exit(EXIT_FAILURE);
879 }
880 initialized[slot] = 0;
881 }
882
883 free(SDDS_input);
884 free(initialized);
885 free(readCode);
886 free(errorMessage);
887 }
888}
889
890void *SDDS_GetParameterMod(SDDS_DATASET *SDDS_dataset, SDDS_DATASET *SDDS_output, char *parameter_name, void *memory) {
891 long index, type, size;
892 void *data;
893 float floatdata;
894 double doubledata;
895 uint64_t ulong64data;
896 int64_t long64data;
897 uint32_t ulongdata;
898 int32_t longdata;
899 unsigned short ushortdata;
900 short shortdata;
901 char chardata;
902 char *stringdata;
903 floatdata = 0.0;
904 doubledata = 0.0;
905 ulong64data = 0;
906 long64data = 0;
907 ulongdata = 0;
908 longdata = 0;
909 ushortdata = 0;
910 shortdata = 0;
911 chardata = '\000';
912 stringdata = "";
913
914 if (!SDDS_CheckDataset(SDDS_dataset, "SDDS_GetParameterMod"))
915 return (NULL);
916 if (!parameter_name) {
917 SDDS_SetError("Unable to get parameter value--parameter name pointer is NULL (SDDS_GetParameterMod)");
918 return (NULL);
919 }
920 if ((index = SDDS_GetParameterIndex(SDDS_dataset, parameter_name)) < 0) {
921 if ((index = SDDS_GetColumnIndex(SDDS_output, parameter_name)) < 0) {
922 SDDS_SetError("Unable to get parameter value--parameter name is unrecognized (SDDS_GetParameterMod)");
923 return (NULL);
924 }
925 if (!(type = SDDS_GetColumnType(SDDS_output, index))) {
926 SDDS_SetError("Unable to get parameter value--parameter data type is invalid (SDDS_GetParameterMod)");
927 return (NULL);
928 }
929 size = SDDS_type_size[type - 1];
930 if (memory)
931 data = memory;
932 else if (!(data = SDDS_Malloc(size))) {
933 SDDS_SetError("Unable to get parameter value--parameter data size is invalid (SDDS_GetParameterMod)");
934 return (NULL);
935 }
936 switch (type) {
937 case SDDS_FLOAT:
938 data = memcpy(data, &floatdata, size);
939 break;
940 case SDDS_DOUBLE:
941 data = memcpy(data, &doubledata, size);
942 break;
943 case SDDS_ULONG64:
944 data = memcpy(data, &ulong64data, size);
945 break;
946 case SDDS_LONG64:
947 data = memcpy(data, &long64data, size);
948 break;
949 case SDDS_ULONG:
950 data = memcpy(data, &ulongdata, size);
951 break;
952 case SDDS_LONG:
953 data = memcpy(data, &longdata, size);
954 break;
955 case SDDS_USHORT:
956 data = memcpy(data, &ushortdata, size);
957 break;
958 case SDDS_SHORT:
959 data = memcpy(data, &shortdata, size);
960 break;
961 case SDDS_CHARACTER:
962 data = memcpy(data, &chardata, size);
963 break;
964 case SDDS_STRING:
965 if (!SDDS_CopyString((char **)data, stringdata))
966 break;
967 }
968 } else {
969 if (!(type = SDDS_GetParameterType(SDDS_dataset, index))) {
970 SDDS_SetError("Unable to get parameter value--parameter data type is invalid (SDDS_GetParameterMod)");
971 return (NULL);
972 }
973 if (!SDDS_dataset->parameter || !SDDS_dataset->parameter[index]) {
974 SDDS_SetError("Unable to get parameter value--parameter data array is NULL (SDDS_GetParameterMod)");
975 return (NULL);
976 }
977 size = SDDS_type_size[type - 1];
978 if (memory)
979 data = memory;
980 else if (!(data = SDDS_Malloc(size))) {
981 SDDS_SetError("Unable to get parameter value--parameter data size is invalid (SDDS_GetParameterMod)");
982 return (NULL);
983 }
984 if (type != SDDS_STRING)
985 memcpy(data, SDDS_dataset->parameter[index], size);
986 else if (!SDDS_CopyString((char **)data, *(char **)SDDS_dataset->parameter[index]))
987 return (NULL);
988 }
989 return (data);
990}
991
992long SDDS_CompareParameterValues(void *param1, void *param2, long type) {
993 double ddiff;
994 int64_t ldiff;
995 char cdiff;
996
997 switch (type) {
998 case SDDS_FLOAT:
999 ddiff = *((float *)param1) - *((float *)param2);
1000 return ddiff < 0 ? -1 : ddiff > 0 ? 1 : 0;
1001 case SDDS_DOUBLE:
1002 ddiff = *((double *)param1) - *((double *)param2);
1003 return ddiff < 0 ? -1 : ddiff > 0 ? 1 : 0;
1004 case SDDS_LONG64:
1005 ldiff = *((int64_t *)param1) - *((int64_t *)param2);
1006 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1007 case SDDS_ULONG64:
1008 ldiff = *((uint64_t *)param1) - *((uint64_t *)param2);
1009 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1010 case SDDS_LONG:
1011 ldiff = *((int32_t *)param1) - *((int32_t *)param2);
1012 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1013 case SDDS_ULONG:
1014 ldiff = *((uint32_t *)param1) - *((uint32_t *)param2);
1015 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1016 case SDDS_SHORT:
1017 ldiff = *((short *)param1) - *((short *)param2);
1018 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1019 case SDDS_USHORT:
1020 ldiff = *((unsigned short *)param1) - *((unsigned short *)param2);
1021 return ldiff < 0 ? -1 : ldiff > 0 ? 1 : 0;
1022 case SDDS_CHARACTER:
1023 cdiff = (short)*((char *)param1) - (short)*((char *)param2);
1024 return cdiff < 0 ? -1 : cdiff > 0 ? 1 : 0;
1025 case SDDS_STRING:
1026 return strcmp(*(char **)param1, *(char **)param2);
1027 default:
1028 SDDS_SetError("Problem doing data comparison--invalid data type (SDDS_CompareParameterValues)");
1029 SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors | SDDS_VERBOSE_PrintErrors);
1030 exit(EXIT_FAILURE);
1031 }
1032}
1033
1034long keep_element(char *name, char **delete, long deletions, char **retain, long retentions) {
1035 long i, flag;
1036
1037 flag = 1;
1038
1039 if (deletions) {
1040 for (i = 0; i < deletions; i++) {
1041 if (wild_match(name, delete[i])) {
1042 flag = 0;
1043 break;
1044 }
1045 }
1046 }
1047
1048 if (retentions) {
1049 if (!deletions)
1050 flag = 0;
1051 for (i = 0; i < retentions; i++) {
1052 if (wild_match(name, retain[i])) {
1053 flag = 1;
1054 break;
1055 }
1056 }
1057 }
1058
1059 return flag;
1060}
SDDS.h
SDDS (Self Describing Data Set) Data Types Definitions and Function Prototypes.
SDDS_ReadRecoveryPossible
int32_t SDDS_ReadRecoveryPossible(SDDS_DATASET *SDDS_dataset)
Checks if any data in an SDDS page was recovered after an error was detected.
Definition SDDS_binary.c:2075
SDDS_CopyAdditionalRows
int32_t SDDS_CopyAdditionalRows(SDDS_DATASET *SDDS_target, SDDS_DATASET *SDDS_source)
Definition SDDS_copy.c:519
SDDS_CopyPage
int32_t SDDS_CopyPage(SDDS_DATASET *SDDS_target, SDDS_DATASET *SDDS_source)
Definition SDDS_copy.c:578
SDDS_type_size
int32_t SDDS_type_size[SDDS_NUM_TYPES]
Array of sizes for each supported data type.
Definition SDDS_data.c:62
SDDS_LengthenTable
int32_t SDDS_LengthenTable(SDDS_DATASET *SDDS_dataset, int64_t n_additional_rows)
Definition SDDS_dataprep.c:303
SDDS_SetRowValues
int32_t SDDS_SetRowValues(SDDS_DATASET *SDDS_dataset, int32_t mode, int64_t row,...)
Definition SDDS_dataprep.c:994
SDDS_StartPage
int32_t SDDS_StartPage(SDDS_DATASET *SDDS_dataset, int64_t expected_n_rows)
Definition SDDS_dataprep.c:70
SDDS_SetParameters
int32_t SDDS_SetParameters(SDDS_DATASET *SDDS_dataset, int32_t mode,...)
Definition SDDS_dataprep.c:375
SDDS_ClearPage
int32_t SDDS_ClearPage(SDDS_DATASET *SDDS_dataset)
Definition SDDS_dataprep.c:194
SDDS_GetParameter
void * SDDS_GetParameter(SDDS_DATASET *SDDS_dataset, char *parameter_name, void *memory)
Retrieves the value of a specified parameter from the current data table of a data set.
Definition SDDS_extract.c:2482
SDDS_GetDescription
int32_t SDDS_GetDescription(SDDS_DATASET *SDDS_dataset, char **text, char **contents)
Retrieves the text and contents descriptions from an SDDS dataset.
Definition SDDS_extract.c:4627
SDDS_ReadPageSparse
int32_t SDDS_ReadPageSparse(SDDS_DATASET *SDDS_dataset, uint32_t mode, int64_t sparse_interval, int64_t sparse_offset, int32_t sparse_statistics)
Definition SDDS_input.c:1083
SDDS_InitializeInput
int32_t SDDS_InitializeInput(SDDS_DATASET *SDDS_dataset, char *filename)
Definition SDDS_input.c:50
SDDS_Terminate
int32_t SDDS_Terminate(SDDS_DATASET *SDDS_dataset)
Definition SDDS_input.c:1471
SDDS_InitializeAppend
int32_t SDDS_InitializeAppend(SDDS_DATASET *SDDS_dataset, const char *filename)
Initializes the SDDS dataset for appending data by adding a new page to an existing file.
Definition SDDS_output.c:288
SDDS_InitializeOutput
int32_t SDDS_InitializeOutput(SDDS_DATASET *SDDS_dataset, int32_t data_mode, int32_t lines_per_row, const char *description, const char *contents, const char *filename)
Initializes the SDDS output dataset.
Definition SDDS_output.c:602
SDDS_InitializeAppendToPage
int32_t SDDS_InitializeAppendToPage(SDDS_DATASET *SDDS_dataset, const char *filename, int64_t updateInterval, int64_t *rowsPresentReturn)
Initializes the SDDS dataset for appending data to the last page of an existing file.
Definition SDDS_output.c:397
SDDS_UpdatePage
int32_t SDDS_UpdatePage(SDDS_DATASET *SDDS_dataset, uint32_t mode)
Updates the current page of the SDDS dataset.
Definition SDDS_output.c:1292
SDDS_WritePage
int32_t SDDS_WritePage(SDDS_DATASET *SDDS_dataset)
Writes the current data table to the output file.
Definition SDDS_output.c:1229
SDDS_DefineColumn
int32_t SDDS_DefineColumn(SDDS_DATASET *SDDS_dataset, const char *name, const char *symbol, const char *units, const char *description, const char *format_string, int32_t type, int32_t field_length)
Defines a data column within the SDDS dataset.
Definition SDDS_output.c:1716
SDDS_WriteLayout
int32_t SDDS_WriteLayout(SDDS_DATASET *SDDS_dataset)
Writes the SDDS layout header to the output file.
Definition SDDS_output.c:900
SDDS_DefineParameter
int32_t SDDS_DefineParameter(SDDS_DATASET *SDDS_dataset, const char *name, const char *symbol, const char *units, const char *description, const char *format_string, int32_t type, char *fixed_value)
Defines a data parameter with a fixed string value.
Definition SDDS_output.c:1473
SDDS_TransferColumnDefinition
int32_t SDDS_TransferColumnDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a column definition from a source dataset to a target dataset.
Definition SDDS_transfer.c:35
SDDS_TransferArrayDefinition
int32_t SDDS_TransferArrayDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers an array definition from a source dataset to a target dataset.
Definition SDDS_transfer.c:111
SDDS_DefineColumnLikeParameter
int32_t SDDS_DefineColumnLikeParameter(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Defines a column in the target dataset based on a parameter definition from the source dataset.
Definition SDDS_transfer.c:264
SDDS_TransferParameterDefinition
int32_t SDDS_TransferParameterDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a parameter definition from a source dataset to a target dataset.
Definition SDDS_transfer.c:73
SDDS_SetError
void SDDS_SetError(char *error_text)
Records an error message in the SDDS error stack.
Definition SDDS_utils.c:421
SDDS_GetParameterType
int32_t SDDS_GetParameterType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a parameter in the SDDS dataset by its index.
Definition SDDS_utils.c:2317
SDDS_GetArrayIndex
int32_t SDDS_GetArrayIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named array in the SDDS dataset.
Definition SDDS_utils.c:1433
SDDS_GetParameterNames
char ** SDDS_GetParameterNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all parameters in the SDDS dataset.
Definition SDDS_utils.c:2567
SDDS_GetParameterIndex
int32_t SDDS_GetParameterIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named parameter in the SDDS dataset.
Definition SDDS_utils.c:1404
SDDS_GetColumnIndex
int32_t SDDS_GetColumnIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named column in the SDDS dataset.
Definition SDDS_utils.c:1375
SDDS_ColumnCount
int32_t SDDS_ColumnCount(SDDS_DATASET *page)
Retrieves the number of columns in the SDDS dataset.
Definition SDDS_utils.c:5098
SDDS_GetColumnNames
char ** SDDS_GetColumnNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all columns in the SDDS dataset.
Definition SDDS_utils.c:2526
SDDS_CheckDataset
int32_t SDDS_CheckDataset(SDDS_DATASET *SDDS_dataset, const char *caller)
Validates the SDDS dataset pointer.
Definition SDDS_utils.c:618
SDDS_PrintErrors
void SDDS_PrintErrors(FILE *fp, int32_t mode)
Prints recorded error messages to a specified file stream.
Definition SDDS_utils.c:474
SDDS_Malloc
void * SDDS_Malloc(size_t size)
Allocates memory of a specified size.
Definition SDDS_utils.c:705
SDDS_DeleteParameterFixedValues
int32_t SDDS_DeleteParameterFixedValues(SDDS_DATASET *SDDS_dataset)
Deletes fixed values from all parameters in the SDDS dataset.
Definition SDDS_utils.c:4974
SDDS_RegisterProgramName
void SDDS_RegisterProgramName(const char *name)
Registers the executable program name for use in error messages.
Definition SDDS_utils.c:318
SDDS_NumberOfErrors
int32_t SDDS_NumberOfErrors()
Retrieves the number of errors recorded by SDDS library routines.
Definition SDDS_utils.c:340
SDDS_GetTypeSize
int32_t SDDS_GetTypeSize(int32_t type)
Retrieves the size in bytes of a specified SDDS data type.
Definition SDDS_utils.c:2374
SDDS_GetColumnType
int32_t SDDS_GetColumnType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a column in the SDDS dataset by its index.
Definition SDDS_utils.c:2219
SDDS_Bomb
void SDDS_Bomb(char *message)
Terminates the program after printing an error message and recorded errors.
Definition SDDS_utils.c:380
SDDS_GetArrayNames
char ** SDDS_GetArrayNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all arrays in the SDDS dataset.
Definition SDDS_utils.c:2605
SDDS_CopyString
int32_t SDDS_CopyString(char **target, const char *source)
Copies a source string to a target string with memory allocation.
Definition SDDS_utils.c:922
SDDS_PrintTypedValue
int32_t SDDS_PrintTypedValue(void *data, int64_t index, int32_t type, char *format, FILE *fp, uint32_t mode)
Prints a data value of a specified type using an optional printf format string.
Definition SDDS_utils.c:67
SDDS_ULONG
#define SDDS_ULONG
Identifier for the unsigned 32-bit integer data type.
Definition SDDStypes.h:67
SDDS_FLOAT
#define SDDS_FLOAT
Identifier for the float data type.
Definition SDDStypes.h:43
SDDS_STRING
#define SDDS_STRING
Identifier for the string data type.
Definition SDDStypes.h:85
SDDS_ULONG64
#define SDDS_ULONG64
Identifier for the unsigned 64-bit integer data type.
Definition SDDStypes.h:55
SDDS_LONG
#define SDDS_LONG
Identifier for the signed 32-bit integer data type.
Definition SDDStypes.h:61
SDDS_SHORT
#define SDDS_SHORT
Identifier for the signed short integer data type.
Definition SDDStypes.h:73
SDDS_CHARACTER
#define SDDS_CHARACTER
Identifier for the character data type.
Definition SDDStypes.h:91
SDDS_USHORT
#define SDDS_USHORT
Identifier for the unsigned short integer data type.
Definition SDDStypes.h:79
SDDS_DOUBLE
#define SDDS_DOUBLE
Identifier for the double data type.
Definition SDDStypes.h:37
SDDS_LONG64
#define SDDS_LONG64
Identifier for the signed 64-bit integer data type.
Definition SDDStypes.h:49
trealloc
void * trealloc(void *old_ptr, uint64_t size_of_block)
Reallocates a memory block to a new size.
Definition array.c:190
tmalloc
void * tmalloc(uint64_t size_of_block)
Allocates a memory block of the specified size with zero initialization.
Definition array.c:65
bomb
void bomb(char *error, char *usage)
Reports error messages to the terminal and aborts the program.
Definition bomb.c:26
fexists
long fexists(const char *filename)
Checks if a file exists.
Definition fexists.c:27
match_string
long match_string(char *string, char **option, long n_options, long mode)
Matches a given string against an array of option strings based on specified modes.
Definition match_string.c:34
scanargs
int scanargs(SCANNED_ARG **scanned, int argc, char **argv)
Definition scanargs.c:36
processPipeOption
long processPipeOption(char **item, long items, unsigned long *flags)
Definition scanargs.c:357
scanItemList
long scanItemList(unsigned long *flags, char **item, long *items, unsigned long mode,...)
Scans a list of items and assigns values based on provided keywords and types.
Definition scanitemlist.c:41
SDDS_DATASET
Definition SDDS.h:308
expand_ranges
char * expand_ranges(char *template)
Expand range specifiers in a wildcard template into explicit character lists.
Definition wild_match.c:429
wild_match
int wild_match(char *string, char *template)
Determine whether one string is a wildcard match for another.
Definition wild_match.c:49
Generated by doxygen 1.12.0