SDDS ToolKit Programs and Libraries for C and Python
Loading...
Searching...
No Matches
Functions
sddsgroupedenvelope.c File Reference

Detailed Description

Create grouped SDDS envelope files without temporary SDDS files.

This program reads multipage SDDS files, groups pages by a selected parameter, and writes one output page per group containing requested row-wise statistics. Accumulators are streaming: by default one input file is open at a time, while -threads uses per-file accumulators that are merged in deterministic input-file order.

Definition in file sddsgroupedenvelope.c.

#include "mdb.h"
#include "SDDS.h"
#include <ctype.h>
#include <errno.h>
#include <float.h>
#include <inttypes.h>
#include <limits.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <glob.h>
#include <strings.h>
#include <unistd.h>

Go to the source code of this file.

Functions

static void * xmalloc (size_t size)
 
static void * xcalloc (size_t count, size_t size)
 
static void * xrealloc (void *ptr, size_t size)
 
static char * xstrdup (const char *s)
 
static void string_list_append_owned (STRING_LIST *list, char *value)
 
static void string_list_append (STRING_LIST *list, const char *value)
 
static void string_list_clear (STRING_LIST *list)
 
static int string_list_contains (const STRING_LIST *list, const char *value)
 
static char * trimmed_copy (const char *start, size_t length)
 
static void split_comma_list (const char *value, STRING_LIST *list)
 
static int is_absolute_path (const char *path)
 
static int is_path_separator (char c)
 
static char * join_path (const char *dir, const char *name)
 
static char * current_directory (void)
 
static int path_exists (const char *path)
 
static int same_existing_file (const char *path1, const char *path2)
 
static int is_directory (const char *path)
 
static const char * base_name (const char *path)
 
static int timestamp_seconds (const char *path)
 
static int compare_input_files (const void *a, const void *b)
 
static int option_name_matches (const char *arg, const char *name, const char **value)
 
static const char * option_value (int *iArg, int argc, char **argv, const char *arg, const char *name)
 
static long parse_long_option (const char *name, const char *value)
 
static const char * stat_suffix (STAT_CODE code)
 
static int stat_uses_weight (STAT_CODE code)
 
static int stat_uses_function_column (STAT_CODE code)
 
static int stat_uses_function_parameter (STAT_CODE code)
 
static void init_options (OPTIONS *opts)
 
static void free_raw_request (RAW_STAT_REQUEST *request)
 
static void free_stat_definition (STAT_DEFINITION *stat)
 
static void free_options (OPTIONS *opts)
 
static RAW_STAT_REQUEST * add_raw_request (OPTIONS *opts, STAT_CODE code)
 
static void add_simple_stat_request (OPTIONS *opts, STAT_CODE code, const char *value)
 
static void add_sum_request (OPTIONS *opts, const char *value)
 
static void add_weighted_request (OPTIONS *opts, STAT_CODE code, const char *value, const char *optionName)
 
static void add_function_request (OPTIONS *opts, STAT_CODE code, const char *value, const char *optionName)
 
static void parse_options (OPTIONS *opts, int argc, char **argv)
 
static void collect_input_files (const char *workDir, const OPTIONS *opts, const char *finalOutput, STRING_LIST *files)
 
static char * stat_column_name (const STAT_DEFINITION *stat)
 
static int result_name_exists (const OPTIONS *opts, const char *name)
 
static int append_stat_definition (OPTIONS *opts, SDDS_DATASET *input, RAW_STAT_REQUEST *request, const char *sourceColumn)
 
static int compile_stat_definitions (OPTIONS *opts, SDDS_DATASET *input)
 
static void free_copy_data (void *data, int32_t type, int64_t rows)
 
static void free_stat_accumulator (STAT_ACCUMULATOR *accumulator, const STAT_DEFINITION *stat, int64_t rows)
 
static void free_group_value (void *value, int32_t type)
 
static void free_group_list (GROUP_LIST *groups, const OPTIONS *opts)
 
static ENVELOPE_GROUP * find_group (GROUP_LIST *groups, const char *groupName)
 
static ENVELOPE_GROUP * add_group (GROUP_LIST *groups, const char *groupName, void *groupValue)
 
static void allocate_stat_accumulator (STAT_ACCUMULATOR *accumulator, const STAT_DEFINITION *stat, int64_t rows)
 
static ROW_ACCUMULATOR * find_row_accumulator (ENVELOPE_GROUP *group, int64_t rows)
 
static ROW_ACCUMULATOR * add_row_accumulator (ENVELOPE_GROUP *group, const OPTIONS *opts, int64_t rows)
 
static ROW_ACCUMULATOR * get_row_accumulator (ENVELOPE_GROUP *group, const OPTIONS *opts, int64_t rows)
 
static ROW_ACCUMULATOR * best_row_accumulator (ENVELOPE_GROUP *group)
 
static int read_parameter_double (SDDS_DATASET *input, const char *name, double *value)
 
static double integer_power (double value, long power)
 
static void accumulate_exmm (STAT_ACCUMULATOR *accumulator, double *data, int64_t rows)
 
static int accumulate_stat (SDDS_DATASET *input, ROW_ACCUMULATOR *rowData, const OPTIONS *opts, long iStat)
 
static int define_stat_column (SDDS_DATASET *output, SDDS_DATASET *input, const STAT_DEFINITION *stat)
 
static int setup_output_file (SDDS_DATASET *output, const char *outputFile, SDDS_DATASET *input, const OPTIONS *opts)
 
static int read_input_file (const char *filename, const OPTIONS *opts, GROUP_LIST *groups)
 
static int read_input_files_serial (const STRING_LIST *files, const OPTIONS *opts, GROUP_LIST *groups)
 
static int read_input_files_threaded (const STRING_LIST *files, const OPTIONS *opts, GROUP_LIST *groups)
 
static int read_input_files (const STRING_LIST *files, const OPTIONS *opts, GROUP_LIST *groups)
 
static void warn_zero_weight (const OPTIONS *opts, const STAT_DEFINITION *stat, int64_t row)
 
static void finalize_stat_values (ROW_ACCUMULATOR *rowData, const OPTIONS *opts, long iStat)
 
static int write_output_pages (const char *outputFile, SDDS_DATASET *templateInput, const OPTIONS *opts, GROUP_LIST *groups)
 
static void warn_for_missing_requested_group_values (const OPTIONS *opts, const GROUP_LIST *groups)
 
static int process_directory (const char *workDir, OPTIONS *opts, int requireInputs)
 
int main (int argc, char **argv)
 

Function Documentation

◆ accumulate_exmm()

static void accumulate_exmm ( STAT_ACCUMULATOR * accumulator,
double * data,
int64_t rows )
static

Definition at line 1233 of file sddsgroupedenvelope.c.

1233 {
1234 int64_t i;
1235 if (!accumulator->initialized) {
1236 for (i = 0; i < rows; i++) {
1237 accumulator->value1[i] = data[i];
1238 accumulator->value2[i] = data[i];
1239 accumulator->value3[i] = data[i];
1240 accumulator->value4[i] = 1;
1241 accumulator->sumWeight[i] = 1;
1242 }
1243 accumulator->initialized = 1;
1244 return;
1245 }
1246 for (i = 0; i < rows; i++) {
1247 accumulator->value1[i] += data[i];
1248 if (data[i] < accumulator->value2[i]) {
1249 accumulator->value2[i] = data[i];
1250 accumulator->value4[i] = 1;
1251 } else if (data[i] == accumulator->value2[i]) {
1252 accumulator->value4[i] += 1;
1253 }
1254 if (data[i] > accumulator->value3[i]) {
1255 accumulator->value3[i] = data[i];
1256 accumulator->sumWeight[i] = 1;
1257 } else if (data[i] == accumulator->value3[i]) {
1258 accumulator->sumWeight[i] += 1;
1259 }
1260 }
1261}

◆ accumulate_stat()

static int accumulate_stat ( SDDS_DATASET * input,
ROW_ACCUMULATOR * rowData,
const OPTIONS * opts,
long iStat )
static

Definition at line 1263 of file sddsgroupedenvelope.c.

1263 {
1264 STAT_DEFINITION *stat = opts->stat + iStat;
1265 STAT_ACCUMULATOR *accumulator = rowData->stat + iStat;
1266 double *data = NULL, *weight = NULL, *otherData = NULL;
1267 double parameterValue = 0;
1268 int64_t i, rows = rowData->rows;
1269
1270 if (stat->code == STAT_COPY) {
1271 if (!accumulator->initialized) {
1272 if (!(accumulator->copyData = SDDS_GetColumn(input, stat->sourceColumn))) {
1273 fprintf(stderr, "error: unable to read copy column %s\n", stat->sourceColumn);
1274 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1275 return 0;
1276 }
1277 accumulator->initialized = 1;
1278 }
1279 return 1;
1280 }
1281
1282 if (!(data = SDDS_GetColumnInDoubles(input, stat->sourceColumn))) {
1283 fprintf(stderr, "error: unable to read column %s\n", stat->sourceColumn);
1284 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1285 return 0;
1286 }
1287 if (stat_uses_weight(stat->code)) {
1288 if (!(weight = SDDS_GetColumnInDoubles(input, stat->weightColumn))) {
1289 fprintf(stderr, "error: unable to read weight column %s\n", stat->weightColumn);
1290 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1291 free(data);
1292 return 0;
1293 }
1294 }
1295 if (stat_uses_function_column(stat->code)) {
1296 if (!(otherData = SDDS_GetColumnInDoubles(input, stat->functionOf))) {
1297 fprintf(stderr, "error: unable to read independent column %s\n", stat->functionOf);
1298 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1299 free(data);
1300 free(weight);
1301 return 0;
1302 }
1303 }
1304 if (stat_uses_function_parameter(stat->code) && !read_parameter_double(input, stat->functionOf, &parameterValue)) {
1305 free(data);
1306 free(weight);
1307 free(otherData);
1308 return 0;
1309 }
1310
1311 switch (stat->code) {
1312 case STAT_MAXIMUM:
1313 if (!accumulator->initialized) {
1314 for (i = 0; i < rows; i++)
1315 accumulator->value1[i] = data[i];
1316 accumulator->initialized = 1;
1317 } else {
1318 for (i = 0; i < rows; i++)
1319 if (accumulator->value1[i] < data[i])
1320 accumulator->value1[i] = data[i];
1321 }
1322 break;
1323 case STAT_MINIMUM:
1324 if (!accumulator->initialized) {
1325 for (i = 0; i < rows; i++)
1326 accumulator->value1[i] = data[i];
1327 accumulator->initialized = 1;
1328 } else {
1329 for (i = 0; i < rows; i++)
1330 if (accumulator->value1[i] > data[i])
1331 accumulator->value1[i] = data[i];
1332 }
1333 break;
1334 case STAT_MEAN:
1335 case STAT_SUM:
1336 for (i = 0; i < rows; i++)
1337 accumulator->value1[i] += stat->code == STAT_SUM ? integer_power(data[i], stat->sumPower) : data[i];
1338 accumulator->initialized = 1;
1339 break;
1340 case STAT_LARGEST:
1341 if (!accumulator->initialized) {
1342 for (i = 0; i < rows; i++)
1343 accumulator->value1[i] = fabs(data[i]);
1344 accumulator->initialized = 1;
1345 } else {
1346 for (i = 0; i < rows; i++)
1347 if (accumulator->value1[i] < fabs(data[i]))
1348 accumulator->value1[i] = fabs(data[i]);
1349 }
1350 break;
1351 case STAT_SIGNED_LARGEST:
1352 if (!accumulator->initialized) {
1353 for (i = 0; i < rows; i++)
1354 accumulator->value1[i] = data[i];
1355 accumulator->initialized = 1;
1356 } else {
1357 for (i = 0; i < rows; i++)
1358 if (fabs(accumulator->value1[i]) < fabs(data[i]))
1359 accumulator->value1[i] = data[i];
1360 }
1361 break;
1362 case STAT_RMS:
1363 for (i = 0; i < rows; i++)
1364 accumulator->value1[i] += data[i] * data[i];
1365 accumulator->initialized = 1;
1366 break;
1367 case STAT_STANDARD_DEVIATION:
1368 case STAT_SIGMA:
1369 for (i = 0; i < rows; i++) {
1370 accumulator->value1[i] += data[i];
1371 accumulator->value2[i] += data[i] * data[i];
1372 }
1373 accumulator->initialized = 1;
1374 break;
1375 case STAT_WMEAN:
1376 for (i = 0; i < rows; i++) {
1377 accumulator->sumWeight[i] += weight[i];
1378 accumulator->value1[i] += data[i] * weight[i];
1379 }
1380 accumulator->initialized = 1;
1381 break;
1382 case STAT_WSTANDARD_DEVIATION:
1383 case STAT_WSIGMA:
1384 for (i = 0; i < rows; i++) {
1385 accumulator->sumWeight[i] += weight[i];
1386 accumulator->value1[i] += data[i] * weight[i];
1387 accumulator->value2[i] += data[i] * data[i] * weight[i];
1388 }
1389 accumulator->initialized = 1;
1390 break;
1391 case STAT_WRMS:
1392 for (i = 0; i < rows; i++) {
1393 accumulator->sumWeight[i] += weight[i];
1394 accumulator->value1[i] += data[i] * data[i] * weight[i];
1395 }
1396 accumulator->initialized = 1;
1397 break;
1398 case STAT_CMAXIMUM:
1399 case STAT_CMINIMUM:
1400 if (!accumulator->initialized) {
1401 for (i = 0; i < rows; i++) {
1402 accumulator->value1[i] = otherData[i];
1403 accumulator->value2[i] = data[i];
1404 }
1405 accumulator->initialized = 1;
1406 } else {
1407 for (i = 0; i < rows; i++) {
1408 if ((stat->code == STAT_CMAXIMUM && accumulator->value2[i] < data[i]) ||
1409 (stat->code == STAT_CMINIMUM && accumulator->value2[i] > data[i])) {
1410 accumulator->value1[i] = otherData[i];
1411 accumulator->value2[i] = data[i];
1412 }
1413 }
1414 }
1415 break;
1416 case STAT_PMAXIMUM:
1417 case STAT_PMINIMUM:
1418 if (!accumulator->initialized) {
1419 for (i = 0; i < rows; i++) {
1420 accumulator->value1[i] = parameterValue;
1421 accumulator->value2[i] = data[i];
1422 }
1423 accumulator->initialized = 1;
1424 } else {
1425 for (i = 0; i < rows; i++) {
1426 if ((stat->code == STAT_PMAXIMUM && accumulator->value2[i] < data[i]) ||
1427 (stat->code == STAT_PMINIMUM && accumulator->value2[i] > data[i])) {
1428 accumulator->value1[i] = parameterValue;
1429 accumulator->value2[i] = data[i];
1430 }
1431 }
1432 }
1433 break;
1434 case STAT_SLOPE:
1435 case STAT_INTERCEPT:
1436 for (i = 0; i < rows; i++) {
1437 accumulator->value1[i] += parameterValue;
1438 accumulator->value2[i] += parameterValue * parameterValue;
1439 accumulator->value3[i] += data[i];
1440 accumulator->value4[i] += parameterValue * data[i];
1441 }
1442 accumulator->initialized = 1;
1443 break;
1444 case STAT_EXMM_MEAN:
1445 accumulate_exmm(accumulator, data, rows);
1446 break;
1447 case STAT_COPY:
1448 break;
1449 }
1450
1451 free(data);
1452 free(weight);
1453 free(otherData);
1454 return 1;
1455}
SDDS_GetColumn
void * SDDS_GetColumn(SDDS_DATASET *SDDS_dataset, char *column_name)
Retrieves a copy of the data for a specified column, including only rows marked as "of interest".
Definition SDDS_extract.c:611
SDDS_GetColumnInDoubles
double * SDDS_GetColumnInDoubles(SDDS_DATASET *SDDS_dataset, char *column_name)
Retrieves the data of a specified numerical column as an array of doubles, considering only rows mark...
Definition SDDS_extract.c:872
SDDS_PrintErrors
void SDDS_PrintErrors(FILE *fp, int32_t mode)
Prints recorded error messages to a specified file stream.
Definition SDDS_utils.c:474
STAT_ACCUMULATOR
Definition sddsgroupedenvelope.c:117
STAT_DEFINITION
Definition sddsenvelope.c:205

◆ add_function_request()

static void add_function_request ( OPTIONS * opts,
STAT_CODE code,
const char * value,
const char * optionName )
static

Definition at line 656 of file sddsgroupedenvelope.c.

656 {
657 STRING_LIST item;
658 RAW_STAT_REQUEST *request;
659 long i;
660
661 memset(&item, 0, sizeof(item));
662 split_comma_list(value, &item);
663 if (item.items < 2) {
664 fprintf(stderr, "error: -%s requires <independent>,<columns>\n", optionName);
665 string_list_clear(&item);
666 exit(EXIT_FAILURE);
667 }
668 request = add_raw_request(opts, code);
669 request->functionOf = xstrdup(item.item[0]);
670 for (i = 1; i < item.items; i++)
671 string_list_append(&request->column, item.item[i]);
672 string_list_clear(&item);
673}
RAW_STAT_REQUEST
Definition sddsgroupedenvelope.c:79
STRING_LIST
Definition sddsgroupedenvelope.c:49

◆ add_group()

static ENVELOPE_GROUP * add_group ( GROUP_LIST * groups,
const char * groupName,
void * groupValue )
static

Definition at line 1122 of file sddsgroupedenvelope.c.

1122 {
1123 ENVELOPE_GROUP *group;
1124 if (groups->groups >= groups->groupsAllocated) {
1125 groups->groupsAllocated = groups->groupsAllocated ? 2 * groups->groupsAllocated : 64;
1126 groups->group = xrealloc(groups->group, sizeof(*groups->group) * groups->groupsAllocated);
1127 }
1128 group = groups->group + groups->groups++;
1129 memset(group, 0, sizeof(*group));
1130 group->name = xstrdup(groupName);
1131 group->value = groupValue;
1132 return group;
1133}
ENVELOPE_GROUP
Definition sddsgroupedenvelope.c:133

◆ add_raw_request()

static RAW_STAT_REQUEST * add_raw_request ( OPTIONS * opts,
STAT_CODE code )
static

Definition at line 595 of file sddsgroupedenvelope.c.

595 {
596 RAW_STAT_REQUEST *request;
597 if (opts->requests >= opts->requestsAllocated) {
598 opts->requestsAllocated = opts->requestsAllocated ? 2 * opts->requestsAllocated : 16;
599 opts->request = xrealloc(opts->request, sizeof(*opts->request) * opts->requestsAllocated);
600 }
601 request = opts->request + opts->requests++;
602 memset(request, 0, sizeof(*request));
603 request->code = code;
604 request->sumPower = 1;
605 return request;
606}

◆ add_row_accumulator()

static ROW_ACCUMULATOR * add_row_accumulator ( ENVELOPE_GROUP * group,
const OPTIONS * opts,
int64_t rows )
static

Definition at line 1182 of file sddsgroupedenvelope.c.

1182 {
1183 ROW_ACCUMULATOR *rowData;
1184 long iStat;
1185
1186 if (group->rowDatas >= group->rowDatasAllocated) {
1187 group->rowDatasAllocated = group->rowDatasAllocated ? 2 * group->rowDatasAllocated : 2;
1188 group->rowData = xrealloc(group->rowData, sizeof(*group->rowData) * group->rowDatasAllocated);
1189 }
1190 rowData = group->rowData + group->rowDatas++;
1191 memset(rowData, 0, sizeof(*rowData));
1192 rowData->rows = rows;
1193 rowData->stat = xcalloc(opts->stats, sizeof(*rowData->stat));
1194 for (iStat = 0; iStat < opts->stats; iStat++)
1195 allocate_stat_accumulator(rowData->stat + iStat, opts->stat + iStat, rows);
1196 return rowData;
1197}
ROW_ACCUMULATOR
Definition sddsgroupedenvelope.c:127

◆ add_simple_stat_request()

static void add_simple_stat_request ( OPTIONS * opts,
STAT_CODE code,
const char * value )
static

Definition at line 608 of file sddsgroupedenvelope.c.

608 {
609 RAW_STAT_REQUEST *request = add_raw_request(opts, code);
610 split_comma_list(value, &request->column);
611}

◆ add_sum_request()

static void add_sum_request ( OPTIONS * opts,
const char * value )
static

Definition at line 613 of file sddsgroupedenvelope.c.

613 {
614 STRING_LIST item;
615 RAW_STAT_REQUEST *request;
616 long i;
617
618 memset(&item, 0, sizeof(item));
619 split_comma_list(value, &item);
620 if (item.items < 2) {
621 fprintf(stderr, "error: -sum requires <power>,<columns>\n");
622 string_list_clear(&item);
623 exit(EXIT_FAILURE);
624 }
625 request = add_raw_request(opts, STAT_SUM);
626 request->sumPower = parse_long_option("sum", item.item[0]);
627 if (request->sumPower < 1) {
628 fprintf(stderr, "error: -sum power must be >= 1\n");
629 string_list_clear(&item);
630 exit(EXIT_FAILURE);
631 }
632 for (i = 1; i < item.items; i++)
633 string_list_append(&request->column, item.item[i]);
634 string_list_clear(&item);
635}

◆ add_weighted_request()

static void add_weighted_request ( OPTIONS * opts,
STAT_CODE code,
const char * value,
const char * optionName )
static

Definition at line 637 of file sddsgroupedenvelope.c.

637 {
638 STRING_LIST item;
639 RAW_STAT_REQUEST *request;
640 long i;
641
642 memset(&item, 0, sizeof(item));
643 split_comma_list(value, &item);
644 if (item.items < 2) {
645 fprintf(stderr, "error: -%s requires <weightColumn>,<columns>\n", optionName);
646 string_list_clear(&item);
647 exit(EXIT_FAILURE);
648 }
649 request = add_raw_request(opts, code);
650 request->weightColumn = xstrdup(item.item[0]);
651 for (i = 1; i < item.items; i++)
652 string_list_append(&request->column, item.item[i]);
653 string_list_clear(&item);
654}

◆ allocate_stat_accumulator()

static void allocate_stat_accumulator ( STAT_ACCUMULATOR * accumulator,
const STAT_DEFINITION * stat,
int64_t rows )
static

Definition at line 1135 of file sddsgroupedenvelope.c.

1135 {
1136 switch (stat->code) {
1137 case STAT_COPY:
1138 break;
1139 case STAT_STANDARD_DEVIATION:
1140 case STAT_SIGMA:
1141 case STAT_CMAXIMUM:
1142 case STAT_CMINIMUM:
1143 case STAT_PMAXIMUM:
1144 case STAT_PMINIMUM:
1145 case STAT_WSTANDARD_DEVIATION:
1146 case STAT_WSIGMA:
1147 accumulator->value1 = xcalloc(rows, sizeof(*accumulator->value1));
1148 accumulator->value2 = xcalloc(rows, sizeof(*accumulator->value2));
1149 if (stat->code == STAT_WSTANDARD_DEVIATION || stat->code == STAT_WSIGMA)
1150 accumulator->sumWeight = xcalloc(rows, sizeof(*accumulator->sumWeight));
1151 break;
1152 case STAT_SLOPE:
1153 case STAT_INTERCEPT:
1154 case STAT_EXMM_MEAN:
1155 accumulator->value1 = xcalloc(rows, sizeof(*accumulator->value1));
1156 accumulator->value2 = xcalloc(rows, sizeof(*accumulator->value2));
1157 accumulator->value3 = xcalloc(rows, sizeof(*accumulator->value3));
1158 accumulator->value4 = xcalloc(rows, sizeof(*accumulator->value4));
1159 if (stat->code == STAT_EXMM_MEAN)
1160 accumulator->sumWeight = xcalloc(rows, sizeof(*accumulator->sumWeight));
1161 break;
1162 case STAT_WMEAN:
1163 case STAT_WRMS:
1164 accumulator->value1 = xcalloc(rows, sizeof(*accumulator->value1));
1165 accumulator->sumWeight = xcalloc(rows, sizeof(*accumulator->sumWeight));
1166 break;
1167 default:
1168 accumulator->value1 = xcalloc(rows, sizeof(*accumulator->value1));
1169 break;
1170 }
1171}

◆ append_stat_definition()

static int append_stat_definition ( OPTIONS * opts,
SDDS_DATASET * input,
RAW_STAT_REQUEST * request,
const char * sourceColumn )
static

Definition at line 910 of file sddsgroupedenvelope.c.

910 {
911 STAT_DEFINITION *stat;
912 char *resultName;
913 int32_t sourceIndex, sourceType;
914
915 if ((sourceIndex = SDDS_GetColumnIndex(input, (char *)sourceColumn)) < 0) {
916 fprintf(stderr, "error: column %s not found\n", sourceColumn);
917 return 0;
918 }
919 sourceType = SDDS_GetColumnType(input, sourceIndex);
920 if (request->code != STAT_COPY && !SDDS_NUMERIC_TYPE(sourceType)) {
921 fprintf(stderr, "error: column %s is not numeric\n", sourceColumn);
922 return 0;
923 }
924 if (stat_uses_weight(request->code)) {
925 int32_t weightIndex, weightType;
926 if ((weightIndex = SDDS_GetColumnIndex(input, request->weightColumn)) < 0) {
927 fprintf(stderr, "error: weight column %s not found\n", request->weightColumn);
928 return 0;
929 }
930 weightType = SDDS_GetColumnType(input, weightIndex);
931 if (!SDDS_NUMERIC_TYPE(weightType)) {
932 fprintf(stderr, "error: weight column %s is not numeric\n", request->weightColumn);
933 return 0;
934 }
935 }
936 if (stat_uses_function_column(request->code)) {
937 int32_t functionIndex, functionType;
938 if ((functionIndex = SDDS_GetColumnIndex(input, request->functionOf)) < 0) {
939 fprintf(stderr, "error: independent column %s not found\n", request->functionOf);
940 return 0;
941 }
942 functionType = SDDS_GetColumnType(input, functionIndex);
943 if (!SDDS_NUMERIC_TYPE(functionType)) {
944 fprintf(stderr, "error: independent column %s is not numeric\n", request->functionOf);
945 return 0;
946 }
947 }
948 if (stat_uses_function_parameter(request->code)) {
949 int32_t functionIndex, functionType;
950 if ((functionIndex = SDDS_GetParameterIndex(input, request->functionOf)) < 0) {
951 fprintf(stderr, "error: independent parameter %s not found\n", request->functionOf);
952 return 0;
953 }
954 functionType = SDDS_GetParameterType(input, functionIndex);
955 if (!SDDS_NUMERIC_TYPE(functionType)) {
956 fprintf(stderr, "error: independent parameter %s is not numeric\n", request->functionOf);
957 return 0;
958 }
959 }
960
961 if (opts->stats >= opts->statsAllocated) {
962 opts->statsAllocated = opts->statsAllocated ? 2 * opts->statsAllocated : 32;
963 opts->stat = xrealloc(opts->stat, sizeof(*opts->stat) * opts->statsAllocated);
964 }
965 stat = opts->stat + opts->stats;
966 memset(stat, 0, sizeof(*stat));
967 stat->code = request->code;
968 stat->sourceColumn = xstrdup(sourceColumn);
969 stat->weightColumn = request->weightColumn ? xstrdup(request->weightColumn) : NULL;
970 stat->functionOf = request->functionOf ? xstrdup(request->functionOf) : NULL;
971 stat->sumPower = request->sumPower;
972 stat->sourceType = sourceType;
973 resultName = stat_column_name(stat);
974 if (result_name_exists(opts, resultName)) {
975 fprintf(stderr, "error: duplicate output column name %s\n", resultName);
976 free(resultName);
977 free_stat_definition(stat);
978 memset(stat, 0, sizeof(*stat));
979 return 0;
980 }
981 stat->resultColumn = resultName;
982 opts->stats++;
983 return 1;
984}
SDDS_GetParameterType
int32_t SDDS_GetParameterType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a parameter in the SDDS dataset by its index.
Definition SDDS_utils.c:2317
SDDS_GetParameterIndex
int32_t SDDS_GetParameterIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named parameter in the SDDS dataset.
Definition SDDS_utils.c:1404
SDDS_GetColumnIndex
int32_t SDDS_GetColumnIndex(SDDS_DATASET *SDDS_dataset, char *name)
Retrieves the index of a named column in the SDDS dataset.
Definition SDDS_utils.c:1375
SDDS_GetColumnType
int32_t SDDS_GetColumnType(SDDS_DATASET *SDDS_dataset, int32_t index)
Retrieves the data type of a column in the SDDS dataset by its index.
Definition SDDS_utils.c:2219
SDDS_NUMERIC_TYPE
#define SDDS_NUMERIC_TYPE(type)
Checks if the given type identifier corresponds to any numeric type.
Definition SDDStypes.h:138

◆ base_name()

static const char * base_name ( const char * path)
static

Definition at line 407 of file sddsgroupedenvelope.c.

407 {
408 const char *slash;
409 slash = strrchr(path, '/');
410#if defined(_WIN32)
411 {
412 const char *backslash = strrchr(path, '\\');
413 if (!slash || (backslash && backslash > slash))
414 slash = backslash;
415 }
416#endif
417 return slash ? slash + 1 : path;
418}

◆ best_row_accumulator()

static ROW_ACCUMULATOR * best_row_accumulator ( ENVELOPE_GROUP * group)
static

Definition at line 1206 of file sddsgroupedenvelope.c.

1206 {
1207 ROW_ACCUMULATOR *best = NULL;
1208 long i;
1209 for (i = 0; i < group->rowDatas; i++) {
1210 if (!best || group->rowData[i].pages > best->pages)
1211 best = group->rowData + i;
1212 }
1213 return best;
1214}

◆ collect_input_files()

static void collect_input_files ( const char * workDir,
const OPTIONS * opts,
const char * finalOutput,
STRING_LIST * files )
static

Definition at line 805 of file sddsgroupedenvelope.c.

805 {
806 long i;
807 for (i = 0; i < opts->pattern.items; i++) {
808 char *globPattern;
809#if defined(_WIN32)
810 WIN32_FIND_DATAA findData;
811 HANDLE findHandle;
812 const char *separator;
813 size_t prefixLength;
814
815 globPattern = is_absolute_path(opts->pattern.item[i]) ? xstrdup(opts->pattern.item[i]) : join_path(workDir, opts->pattern.item[i]);
816 separator = strrchr(globPattern, '/');
817 {
818 const char *backslash = strrchr(globPattern, '\\');
819 if (!separator || (backslash && backslash > separator))
820 separator = backslash;
821 }
822 prefixLength = separator ? (size_t)(separator - globPattern + 1) : 0;
823 findHandle = FindFirstFileA(globPattern, &findData);
824 if (findHandle != INVALID_HANDLE_VALUE) {
825 do {
826 char *path;
827 size_t nameLength;
828 if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
829 continue;
830 nameLength = strlen(findData.cFileName);
831 path = xmalloc(prefixLength + nameLength + 1);
832 memcpy(path, globPattern, prefixLength);
833 memcpy(path + prefixLength, findData.cFileName, nameLength + 1);
834 if ((!finalOutput || !same_existing_file(path, finalOutput)) &&
835 !string_list_contains(files, path))
836 string_list_append_owned(files, path);
837 else
838 free(path);
839 } while (FindNextFileA(findHandle, &findData));
840 if (GetLastError() != ERROR_NO_MORE_FILES)
841 fprintf(stderr, "warning: file search failed for %s\n", globPattern);
842 FindClose(findHandle);
843 } else if (GetLastError() != ERROR_FILE_NOT_FOUND &&
844 GetLastError() != ERROR_PATH_NOT_FOUND) {
845 fprintf(stderr, "warning: file search failed for %s\n", globPattern);
846 }
847 free(globPattern);
848#else
849 glob_t globResult;
850 int status;
851 size_t j;
852
853 globPattern = is_absolute_path(opts->pattern.item[i]) ? xstrdup(opts->pattern.item[i]) : join_path(workDir, opts->pattern.item[i]);
854 memset(&globResult, 0, sizeof(globResult));
855 status = glob(globPattern, 0, NULL, &globResult);
856 if (status == 0) {
857 for (j = 0; j < globResult.gl_pathc; j++) {
858 const char *path = globResult.gl_pathv[j];
859 if (finalOutput && same_existing_file(path, finalOutput))
860 continue;
861 if (!string_list_contains(files, path))
862 string_list_append(files, path);
863 }
864 } else if (status != GLOB_NOMATCH) {
865 fprintf(stderr, "warning: glob failed for %s\n", globPattern);
866 }
867 globfree(&globResult);
868 free(globPattern);
869#endif
870 }
871 if (files->items > 1)
872 qsort(files->item, files->items, sizeof(*files->item), compare_input_files);
873}

◆ compare_input_files()

static int compare_input_files ( const void * a,
const void * b )
static

Definition at line 435 of file sddsgroupedenvelope.c.

435 {
436 const char *pa = *(const char * const *)a;
437 const char *pb = *(const char * const *)b;
438 int ta = timestamp_seconds(pa);
439 int tb = timestamp_seconds(pb);
440 int cmp;
441
442 if (ta >= 0 && tb >= 0 && ta != tb)
443 return ta < tb ? -1 : 1;
444 cmp = strcmp(base_name(pa), base_name(pb));
445 if (cmp)
446 return cmp;
447 return strcmp(pa, pb);
448}

◆ compile_stat_definitions()

static int compile_stat_definitions ( OPTIONS * opts,
SDDS_DATASET * input )
static

Definition at line 986 of file sddsgroupedenvelope.c.

986 {
987 long iReq, iCol;
988 int32_t groupIndex;
989
990 if ((groupIndex = SDDS_GetParameterIndex(input, opts->groupBy)) < 0) {
991 fprintf(stderr, "error: input file does not contain grouping parameter %s\n", opts->groupBy);
992 return 0;
993 }
994 opts->groupByType = SDDS_GetParameterType(input, groupIndex);
995 if (opts->groupByType <= 0) {
996 fprintf(stderr, "error: grouping parameter %s has invalid type\n", opts->groupBy);
997 return 0;
998 }
999
1000 for (iReq = 0; iReq < opts->requests; iReq++) {
1001 RAW_STAT_REQUEST *request = opts->request + iReq;
1002 for (iCol = 0; iCol < request->column.items; iCol++) {
1003 char *columnPattern = request->column.item[iCol];
1004 if (has_wildcards(columnPattern)) {
1005 char **columnName = NULL;
1006 int32_t columnNames = 0;
1007 int32_t iName;
1008 SDDS_SetColumnFlags(input, 0);
1009 if (!SDDS_SetColumnsOfInterest(input, SDDS_MATCH_STRING, columnPattern, SDDS_OR) ||
1010 !(columnName = SDDS_GetColumnNames(input, &columnNames)) || columnNames <= 0) {
1011 fprintf(stderr, "error: no columns selected for wildcard %s\n", columnPattern);
1012 free(columnName);
1013 return 0;
1014 }
1015 for (iName = 0; iName < columnNames; iName++) {
1016 if (!append_stat_definition(opts, input, request, columnName[iName])) {
1017 free(columnName);
1018 return 0;
1019 }
1020 }
1021 free(columnName);
1022 } else if (!append_stat_definition(opts, input, request, columnPattern)) {
1023 return 0;
1024 }
1025 }
1026 }
1027 if (opts->stats == 0) {
1028 fprintf(stderr, "error: no statistic columns were selected\n");
1029 return 0;
1030 }
1031 return 1;
1032}
SDDS_SetColumnsOfInterest
int32_t SDDS_SetColumnsOfInterest(SDDS_DATASET *SDDS_dataset, int32_t mode,...)
Sets the acceptance flags for columns based on specified naming criteria.
Definition SDDS_extract.c:432
SDDS_SetColumnFlags
int32_t SDDS_SetColumnFlags(SDDS_DATASET *SDDS_dataset, int32_t column_flag_value)
Sets the acceptance flags for all columns in the current data table of a data set.
Definition SDDS_extract.c:218
SDDS_GetColumnNames
char ** SDDS_GetColumnNames(SDDS_DATASET *SDDS_dataset, int32_t *number)
Retrieves the names of all columns in the SDDS dataset.
Definition SDDS_utils.c:2526
has_wildcards
int has_wildcards(char *template)
Check if a template string contains any wildcard characters.
Definition wild_match.c:498

◆ current_directory()

static char * current_directory ( void )
static

Definition at line 349 of file sddsgroupedenvelope.c.

349 {
350 char buffer[PATH_MAX];
351 if (!getcwd(buffer, sizeof(buffer))) {
352 fprintf(stderr, "error: unable to determine current directory: %s\n", strerror(errno));
353 exit(EXIT_FAILURE);
354 }
355 return xstrdup(buffer);
356}

◆ define_stat_column()

static int define_stat_column ( SDDS_DATASET * output,
SDDS_DATASET * input,
const STAT_DEFINITION * stat )
static

Definition at line 1610 of file sddsgroupedenvelope.c.

1610 {
1611 char *symbol = NULL;
1612 char *newSymbol;
1613 size_t needed;
1614 const char *definitionColumn = stat->sourceColumn;
1615
1616 if (stat->code == STAT_CMAXIMUM || stat->code == STAT_CMINIMUM)
1617 definitionColumn = stat->functionOf;
1618
1619 if (stat->code == STAT_COPY) {
1620 if (!SDDS_TransferColumnDefinition(output, input, stat->sourceColumn, stat->resultColumn))
1621 return 0;
1622 return 1;
1623 }
1624 if (!SDDS_TransferColumnDefinition(output, input, (char *)definitionColumn, stat->resultColumn))
1625 return 0;
1626 if (SDDS_ChangeColumnInformation(output, "description", NULL, SDDS_SET_BY_NAME, stat->resultColumn) != SDDS_STRING) {
1627 fprintf(stderr, "error: unable to clear description for output column %s\n", stat->resultColumn);
1628 return 0;
1629 }
1630 if (SDDS_ChangeColumnInformation(output, "type", "double", SDDS_PASS_BY_STRING | SDDS_SET_BY_NAME, stat->resultColumn) != SDDS_LONG) {
1631 fprintf(stderr, "error: unable to set output column %s type to double\n", stat->resultColumn);
1632 return 0;
1633 }
1634 if (SDDS_GetColumnInformation(output, "symbol", &symbol, SDDS_BY_NAME, stat->resultColumn) != SDDS_STRING) {
1635 fprintf(stderr, "error: unable to read symbol for output column %s\n", stat->resultColumn);
1636 return 0;
1637 }
1638 if (!symbol || !strlen(symbol)) {
1639 free(symbol);
1640 symbol = xstrdup(stat->sourceColumn);
1641 }
1642
1643 needed = strlen(stat_suffix(stat->code)) + strlen(symbol) + 3;
1644 newSymbol = xmalloc(needed);
1645 snprintf(newSymbol, needed, "%s[%s]", stat_suffix(stat->code), symbol);
1646 free(symbol);
1647
1648 if (SDDS_ChangeColumnInformation(output, "symbol", newSymbol, SDDS_BY_NAME, stat->resultColumn) != SDDS_STRING) {
1649 fprintf(stderr, "error: unable to set symbol for output column %s\n", stat->resultColumn);
1650 free(newSymbol);
1651 return 0;
1652 }
1653 free(newSymbol);
1654 return 1;
1655}
SDDS_ChangeColumnInformation
int32_t SDDS_ChangeColumnInformation(SDDS_DATASET *SDDS_dataset, char *field_name, void *memory, int32_t mode,...)
Modifies a specific field in a column definition within the SDDS dataset.
Definition SDDS_info.c:364
SDDS_GetColumnInformation
int32_t SDDS_GetColumnInformation(SDDS_DATASET *SDDS_dataset, char *field_name, void *memory, int32_t mode,...)
Retrieves information about a specified column in the SDDS dataset.
Definition SDDS_info.c:41
SDDS_TransferColumnDefinition
int32_t SDDS_TransferColumnDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a column definition from a source dataset to a target dataset.
Definition SDDS_transfer.c:35
SDDS_STRING
#define SDDS_STRING
Identifier for the string data type.
Definition SDDStypes.h:85
SDDS_LONG
#define SDDS_LONG
Identifier for the signed 32-bit integer data type.
Definition SDDStypes.h:61

◆ finalize_stat_values()

static void finalize_stat_values ( ROW_ACCUMULATOR * rowData,
const OPTIONS * opts,
long iStat )
static

Definition at line 1842 of file sddsgroupedenvelope.c.

1842 {
1843 STAT_DEFINITION *stat = opts->stat + iStat;
1844 STAT_ACCUMULATOR *accumulator = rowData->stat + iStat;
1845 int64_t i, pages = rowData->pages;
1846
1847 switch (stat->code) {
1848 case STAT_COPY:
1849 case STAT_MAXIMUM:
1850 case STAT_MINIMUM:
1851 case STAT_LARGEST:
1852 case STAT_SIGNED_LARGEST:
1853 case STAT_SUM:
1854 case STAT_CMAXIMUM:
1855 case STAT_CMINIMUM:
1856 case STAT_PMAXIMUM:
1857 case STAT_PMINIMUM:
1858 break;
1859 case STAT_MEAN:
1860 for (i = 0; i < rowData->rows; i++)
1861 accumulator->value1[i] /= pages;
1862 break;
1863 case STAT_RMS:
1864 for (i = 0; i < rowData->rows; i++)
1865 accumulator->value1[i] = sqrt(accumulator->value1[i] / pages);
1866 break;
1867 case STAT_STANDARD_DEVIATION:
1868 if (pages < 2) {
1869 for (i = 0; i < rowData->rows; i++)
1870 accumulator->value1[i] = DBL_MAX;
1871 } else {
1872 for (i = 0; i < rowData->rows; i++) {
1873 double variance = accumulator->value2[i] / pages - sqr(accumulator->value1[i] / pages);
1874 accumulator->value1[i] = variance <= 0 ? 0 : sqrt(variance * pages / (pages - 1.0));
1875 }
1876 }
1877 break;
1878 case STAT_SIGMA:
1879 if (pages < 2) {
1880 for (i = 0; i < rowData->rows; i++)
1881 accumulator->value1[i] = DBL_MAX;
1882 } else {
1883 for (i = 0; i < rowData->rows; i++) {
1884 double variance = accumulator->value2[i] / pages - sqr(accumulator->value1[i] / pages);
1885 accumulator->value1[i] = variance <= 0 ? 0 : sqrt(variance / (pages - 1.0));
1886 }
1887 }
1888 break;
1889 case STAT_WMEAN:
1890 for (i = 0; i < rowData->rows; i++) {
1891 if (accumulator->sumWeight[i])
1892 accumulator->value1[i] /= accumulator->sumWeight[i];
1893 else {
1894 warn_zero_weight(opts, stat, i);
1895 accumulator->value1[i] = DBL_MAX;
1896 }
1897 }
1898 break;
1899 case STAT_WSTANDARD_DEVIATION:
1900 if (pages < 2) {
1901 for (i = 0; i < rowData->rows; i++)
1902 accumulator->value1[i] = DBL_MAX;
1903 } else {
1904 for (i = 0; i < rowData->rows; i++) {
1905 if (accumulator->sumWeight[i]) {
1906 double mean = accumulator->value1[i] / accumulator->sumWeight[i];
1907 double variance = accumulator->value2[i] / accumulator->sumWeight[i] - mean * mean;
1908 accumulator->value1[i] = variance <= 0 ? 0 : sqrt(variance * pages / (pages - 1.0));
1909 } else {
1910 warn_zero_weight(opts, stat, i);
1911 accumulator->value1[i] = DBL_MAX;
1912 }
1913 }
1914 }
1915 break;
1916 case STAT_WRMS:
1917 for (i = 0; i < rowData->rows; i++) {
1918 if (accumulator->sumWeight[i])
1919 accumulator->value1[i] = sqrt(accumulator->value1[i] / accumulator->sumWeight[i]);
1920 else {
1921 warn_zero_weight(opts, stat, i);
1922 accumulator->value1[i] = DBL_MAX;
1923 }
1924 }
1925 break;
1926 case STAT_WSIGMA:
1927 if (pages < 2) {
1928 for (i = 0; i < rowData->rows; i++)
1929 accumulator->value1[i] = DBL_MAX;
1930 } else {
1931 for (i = 0; i < rowData->rows; i++) {
1932 if (accumulator->sumWeight[i]) {
1933 double mean = accumulator->value1[i] / accumulator->sumWeight[i];
1934 double variance = accumulator->value2[i] / accumulator->sumWeight[i] - mean * mean;
1935 accumulator->value1[i] = variance <= 0 ? 0 : sqrt(variance / (pages - 1.0));
1936 } else {
1937 warn_zero_weight(opts, stat, i);
1938 accumulator->value1[i] = DBL_MAX;
1939 }
1940 }
1941 }
1942 break;
1943 case STAT_SLOPE:
1944 for (i = 0; i < rowData->rows; i++) {
1945 double D = pages * accumulator->value2[i] - accumulator->value1[i] * accumulator->value1[i];
1946 accumulator->value1[i] = D ? (pages * accumulator->value4[i] - accumulator->value1[i] * accumulator->value3[i]) / D : DBL_MAX;
1947 }
1948 break;
1949 case STAT_INTERCEPT:
1950 for (i = 0; i < rowData->rows; i++) {
1951 double D = pages * accumulator->value2[i] - accumulator->value1[i] * accumulator->value1[i];
1952 accumulator->value1[i] = D ? (accumulator->value2[i] * accumulator->value3[i] - accumulator->value1[i] * accumulator->value4[i]) / D : DBL_MAX;
1953 }
1954 break;
1955 case STAT_EXMM_MEAN:
1956 for (i = 0; i < rowData->rows; i++) {
1957 double min = accumulator->value2[i];
1958 double max = accumulator->value3[i];
1959 double excluded = min == max ? (double)pages : accumulator->value4[i] + accumulator->sumWeight[i];
1960 double kept = (double)pages - excluded;
1961 if (kept <= 0)
1962 accumulator->value1[i] = min;
1963 else
1964 accumulator->value1[i] = (accumulator->value1[i] - min * accumulator->value4[i] - max * accumulator->sumWeight[i]) / kept;
1965 }
1966 break;
1967 }
1968}

◆ find_group()

static ENVELOPE_GROUP * find_group ( GROUP_LIST * groups,
const char * groupName )
static

Definition at line 1113 of file sddsgroupedenvelope.c.

1113 {
1114 long i;
1115 for (i = 0; i < groups->groups; i++) {
1116 if (strcmp(groups->group[i].name, groupName) == 0)
1117 return groups->group + i;
1118 }
1119 return NULL;
1120}

◆ find_row_accumulator()

static ROW_ACCUMULATOR * find_row_accumulator ( ENVELOPE_GROUP * group,
int64_t rows )
static

Definition at line 1173 of file sddsgroupedenvelope.c.

1173 {
1174 long i;
1175 for (i = 0; i < group->rowDatas; i++) {
1176 if (group->rowData[i].rows == rows)
1177 return group->rowData + i;
1178 }
1179 return NULL;
1180}

◆ free_copy_data()

static void free_copy_data ( void * data,
int32_t type,
int64_t rows )
static

Definition at line 1034 of file sddsgroupedenvelope.c.

1034 {
1035 int64_t i;
1036 if (!data)
1037 return;
1038 if (type == SDDS_STRING) {
1039 char **stringData = (char **)data;
1040 for (i = 0; i < rows; i++)
1041 free(stringData[i]);
1042 }
1043 free(data);
1044}

◆ free_group_list()

static void free_group_list ( GROUP_LIST * groups,
const OPTIONS * opts )
static

Definition at line 1088 of file sddsgroupedenvelope.c.

1088 {
1089 long i, j, iStat;
1090 for (i = 0; i < groups->groups; i++) {
1091 ENVELOPE_GROUP *group = groups->group + i;
1092 free(group->name);
1093 free_group_value(group->value, opts->groupByType);
1094 for (j = 0; j < group->rowDatas; j++) {
1095 ROW_ACCUMULATOR *rowData = group->rowData + j;
1096 for (iStat = 0; iStat < opts->stats; iStat++)
1097 free_stat_accumulator(rowData->stat + iStat, opts->stat + iStat, rowData->rows);
1098 free(rowData->stat);
1099 }
1100 free(group->rowData);
1101 }
1102 free(groups->group);
1103 groups->group = NULL;
1104 groups->groups = groups->groupsAllocated = 0;
1105}

◆ free_group_value()

static void free_group_value ( void * value,
int32_t type )
static

Definition at line 1056 of file sddsgroupedenvelope.c.

1056 {
1057 if (!value)
1058 return;
1059 if (type == SDDS_STRING)
1060 free(*(char **)value);
1061 free(value);
1062}

◆ free_options()

static void free_options ( OPTIONS * opts)
static

Definition at line 580 of file sddsgroupedenvelope.c.

580 {
581 long i;
582 string_list_clear(&opts->pattern);
583 string_list_clear(&opts->groupValue);
584 free(opts->groupBy);
585 free(opts->inputDir);
586 free(opts->output);
587 for (i = 0; i < opts->requests; i++)
588 free_raw_request(opts->request + i);
589 free(opts->request);
590 for (i = 0; i < opts->stats; i++)
591 free_stat_definition(opts->stat + i);
592 free(opts->stat);
593}

◆ free_raw_request()

static void free_raw_request ( RAW_STAT_REQUEST * request)
static

Definition at line 567 of file sddsgroupedenvelope.c.

567 {
568 string_list_clear(&request->column);
569 free(request->weightColumn);
570 free(request->functionOf);
571}

◆ free_stat_accumulator()

static void free_stat_accumulator ( STAT_ACCUMULATOR * accumulator,
const STAT_DEFINITION * stat,
int64_t rows )
static

Definition at line 1046 of file sddsgroupedenvelope.c.

1046 {
1047 if (stat->code == STAT_COPY)
1048 free_copy_data(accumulator->copyData, stat->sourceType, rows);
1049 free(accumulator->value1);
1050 free(accumulator->value2);
1051 free(accumulator->value3);
1052 free(accumulator->value4);
1053 free(accumulator->sumWeight);
1054}

◆ free_stat_definition()

static void free_stat_definition ( STAT_DEFINITION * stat)
static

Definition at line 573 of file sddsgroupedenvelope.c.

573 {
574 free(stat->sourceColumn);
575 free(stat->resultColumn);
576 free(stat->weightColumn);
577 free(stat->functionOf);
578}

◆ get_row_accumulator()

static ROW_ACCUMULATOR * get_row_accumulator ( ENVELOPE_GROUP * group,
const OPTIONS * opts,
int64_t rows )
static

Definition at line 1199 of file sddsgroupedenvelope.c.

1199 {
1200 ROW_ACCUMULATOR *rowData = find_row_accumulator(group, rows);
1201 if (rowData)
1202 return rowData;
1203 return add_row_accumulator(group, opts, rows);
1204}

◆ init_options()

static void init_options ( OPTIONS * opts)
static

Definition at line 555 of file sddsgroupedenvelope.c.

555 {
556 memset(opts, 0, sizeof(*opts));
557 string_list_append(&opts->pattern, "PSD-*.xz");
558 string_list_append(&opts->pattern, "PSD-*.gz");
559 opts->groupBy = xstrdup("PSName");
560 opts->inputDir = current_directory();
561 opts->output = xstrdup("PSD-envelope.sdds");
562 opts->overwrite = 1;
563 opts->threads = 1;
564 opts->columnMajorOrder = 1;
565}

◆ integer_power()

static double integer_power ( double value,
long power )
static

Definition at line 1225 of file sddsgroupedenvelope.c.

1225 {
1226 double result = 1;
1227 long i;
1228 for (i = 0; i < power; i++)
1229 result *= value;
1230 return result;
1231}

◆ is_absolute_path()

static int is_absolute_path ( const char * path)
static

Definition at line 307 of file sddsgroupedenvelope.c.

307 {
308 if (!path || !path[0])
309 return 0;
310 if (path[0] == '/')
311 return 1;
312#if defined(_WIN32)
313 if (path[0] == '\\')
314 return 1;
315#endif
316 if (isalpha((unsigned char)path[0]) && path[1] == ':')
317 return 1;
318 return 0;
319}

◆ is_directory()

static int is_directory ( const char * path)
static

Definition at line 402 of file sddsgroupedenvelope.c.

402 {
403 struct stat st;
404 return stat(path, &st) == 0 && S_ISDIR(st.st_mode);
405}

◆ is_path_separator()

static int is_path_separator ( char c)
static

Definition at line 321 of file sddsgroupedenvelope.c.

321 {
322#if defined(_WIN32)
323 return c == '/' || c == '\\';
324#else
325 return c == '/';
326#endif
327}

◆ join_path()

static char * join_path ( const char * dir,
const char * name )
static

Definition at line 329 of file sddsgroupedenvelope.c.

329 {
330 size_t dirLen, nameLen, needSlash;
331 char *path;
332
333 if (is_absolute_path(name))
334 return xstrdup(name);
335 if (!dir || !dir[0])
336 return xstrdup(name);
337
338 dirLen = strlen(dir);
339 nameLen = strlen(name);
340 needSlash = dirLen && !is_path_separator(dir[dirLen - 1]);
341 path = xmalloc(dirLen + needSlash + nameLen + 1);
342 memcpy(path, dir, dirLen);
343 if (needSlash)
344 path[dirLen++] = '/';
345 memcpy(path + dirLen, name, nameLen + 1);
346 return path;
347}

◆ main()

int main ( int argc,
char ** argv )

Definition at line 2127 of file sddsgroupedenvelope.c.

2127 {
2128 OPTIONS opts;
2129 int ok;
2130
2131 SDDS_RegisterProgramName(argv[0]);
2132 if (argc == 1) {
2133 fputs(USAGE, stderr);
2134 return EXIT_FAILURE;
2135 }
2136
2137 init_options(&opts);
2138 parse_options(&opts, argc, argv);
2139
2140 ok = process_directory(opts.inputDir, &opts, 1) == PROCESS_DONE;
2141
2142 free_options(&opts);
2143 return ok ? EXIT_SUCCESS : EXIT_FAILURE;
2144}
SDDS_RegisterProgramName
void SDDS_RegisterProgramName(const char *name)
Registers the executable program name for use in error messages.
Definition SDDS_utils.c:318

◆ option_name_matches()

static int option_name_matches ( const char * arg,
const char * name,
const char ** value )
static

Definition at line 450 of file sddsgroupedenvelope.c.

450 {
451 size_t nameLen;
452 const char *body, *eq;
453
454 if (!arg || arg[0] != '-')
455 return 0;
456 body = arg + 1;
457 if (body[0] == '-')
458 body++;
459 eq = strchr(body, '=');
460 nameLen = eq ? (size_t)(eq - body) : strlen(body);
461 if (strlen(name) != nameLen || strncasecmp(body, name, nameLen) != 0)
462 return 0;
463 if (value)
464 *value = eq ? eq + 1 : NULL;
465 return 1;
466}

◆ option_value()

static const char * option_value ( int * iArg,
int argc,
char ** argv,
const char * arg,
const char * name )
static

Definition at line 468 of file sddsgroupedenvelope.c.

468 {
469 const char *value = NULL;
470 if (!option_name_matches(arg, name, &value))
471 return NULL;
472 if (value)
473 return value;
474 if (*iArg + 1 >= argc) {
475 fprintf(stderr, "error: missing value for -%s\n", name);
476 exit(EXIT_FAILURE);
477 }
478 return argv[++(*iArg)];
479}

◆ parse_long_option()

static long parse_long_option ( const char * name,
const char * value )
static

Definition at line 481 of file sddsgroupedenvelope.c.

481 {
482 char *endptr;
483 long result;
484 errno = 0;
485 result = strtol(value, &endptr, 10);
486 if (errno || !value[0] || *endptr) {
487 fprintf(stderr, "error: -%s must be an integer\n", name);
488 exit(EXIT_FAILURE);
489 }
490 return result;
491}

◆ parse_options()

static void parse_options ( OPTIONS * opts,
int argc,
char ** argv )
static

Definition at line 675 of file sddsgroupedenvelope.c.

675 {
676 int iArg, patternSpecified = 0;
677
678 for (iArg = 1; iArg < argc; iArg++) {
679 const char *arg = argv[iArg];
680 const char *value;
681
682 if (option_name_matches(arg, "help", NULL) || option_name_matches(arg, "h", NULL)) {
683 fputs(USAGE, stdout);
684 exit(EXIT_SUCCESS);
685 } else if ((value = option_value(&iArg, argc, argv, arg, "pattern"))) {
686 if (!patternSpecified) {
687 string_list_clear(&opts->pattern);
688 patternSpecified = 1;
689 }
690 string_list_append(&opts->pattern, value);
691 } else if ((value = option_value(&iArg, argc, argv, arg, "inputDir"))) {
692 if (!strlen(value)) {
693 fprintf(stderr, "error: -inputDir value may not be blank\n");
694 exit(EXIT_FAILURE);
695 }
696 free(opts->inputDir);
697 opts->inputDir = xstrdup(value);
698 } else if ((value = option_value(&iArg, argc, argv, arg, "groupBy"))) {
699 if (!strlen(value)) {
700 fprintf(stderr, "error: -groupBy value may not be blank\n");
701 exit(EXIT_FAILURE);
702 }
703 free(opts->groupBy);
704 opts->groupBy = xstrdup(value);
705 } else if ((value = option_value(&iArg, argc, argv, arg, "groupValue"))) {
706 string_list_append(&opts->groupValue, value);
707 } else if ((value = option_value(&iArg, argc, argv, arg, "output"))) {
708 free(opts->output);
709 opts->output = xstrdup(value);
710 } else if ((value = option_value(&iArg, argc, argv, arg, "majorOrder"))) {
711 if (strcasecmp(value, "column") == 0)
712 opts->columnMajorOrder = 1;
713 else if (strcasecmp(value, "row") == 0)
714 opts->columnMajorOrder = 0;
715 else {
716 fprintf(stderr, "error: -majorOrder must be row or column\n");
717 exit(EXIT_FAILURE);
718 }
719 } else if ((value = option_value(&iArg, argc, argv, arg, "threads"))) {
720 long threads = parse_long_option("threads", value);
721 if (threads < 1) {
722 fprintf(stderr, "error: -threads must be >= 1\n");
723 exit(EXIT_FAILURE);
724 }
725 if (threads > INT_MAX) {
726 fprintf(stderr, "error: -threads value is too large\n");
727 exit(EXIT_FAILURE);
728 }
729 opts->threads = (int)threads;
730 } else if ((value = option_value(&iArg, argc, argv, arg, "copy"))) {
731 add_simple_stat_request(opts, STAT_COPY, value);
732 } else if ((value = option_value(&iArg, argc, argv, arg, "maximum"))) {
733 add_simple_stat_request(opts, STAT_MAXIMUM, value);
734 } else if ((value = option_value(&iArg, argc, argv, arg, "minimum"))) {
735 add_simple_stat_request(opts, STAT_MINIMUM, value);
736 } else if ((value = option_value(&iArg, argc, argv, arg, "mean"))) {
737 add_simple_stat_request(opts, STAT_MEAN, value);
738 } else if ((value = option_value(&iArg, argc, argv, arg, "largest"))) {
739 add_simple_stat_request(opts, STAT_LARGEST, value);
740 } else if ((value = option_value(&iArg, argc, argv, arg, "signedLargest"))) {
741 add_simple_stat_request(opts, STAT_SIGNED_LARGEST, value);
742 } else if ((value = option_value(&iArg, argc, argv, arg, "sum"))) {
743 add_sum_request(opts, value);
744 } else if ((value = option_value(&iArg, argc, argv, arg, "rms"))) {
745 add_simple_stat_request(opts, STAT_RMS, value);
746 } else if ((value = option_value(&iArg, argc, argv, arg, "standarddeviations")) ||
747 (value = option_value(&iArg, argc, argv, arg, "standarddeviation"))) {
748 add_simple_stat_request(opts, STAT_STANDARD_DEVIATION, value);
749 } else if ((value = option_value(&iArg, argc, argv, arg, "sigmas")) ||
750 (value = option_value(&iArg, argc, argv, arg, "sigma"))) {
751 add_simple_stat_request(opts, STAT_SIGMA, value);
752 } else if ((value = option_value(&iArg, argc, argv, arg, "wmean"))) {
753 add_weighted_request(opts, STAT_WMEAN, value, "wmean");
754 } else if ((value = option_value(&iArg, argc, argv, arg, "wstandarddeviations")) ||
755 (value = option_value(&iArg, argc, argv, arg, "wstandarddeviation"))) {
756 add_weighted_request(opts, STAT_WSTANDARD_DEVIATION, value, "wstandarddeviations");
757 } else if ((value = option_value(&iArg, argc, argv, arg, "wrms"))) {
758 add_weighted_request(opts, STAT_WRMS, value, "wrms");
759 } else if ((value = option_value(&iArg, argc, argv, arg, "wsigma")) ||
760 (value = option_value(&iArg, argc, argv, arg, "wsigmas"))) {
761 add_weighted_request(opts, STAT_WSIGMA, value, "wsigma");
762 } else if ((value = option_value(&iArg, argc, argv, arg, "cmaximum"))) {
763 add_function_request(opts, STAT_CMAXIMUM, value, "cmaximum");
764 } else if ((value = option_value(&iArg, argc, argv, arg, "cminimum"))) {
765 add_function_request(opts, STAT_CMINIMUM, value, "cminimum");
766 } else if ((value = option_value(&iArg, argc, argv, arg, "pmaximum"))) {
767 add_function_request(opts, STAT_PMAXIMUM, value, "pmaximum");
768 } else if ((value = option_value(&iArg, argc, argv, arg, "pminimum"))) {
769 add_function_request(opts, STAT_PMINIMUM, value, "pminimum");
770 } else if ((value = option_value(&iArg, argc, argv, arg, "slope"))) {
771 add_function_request(opts, STAT_SLOPE, value, "slope");
772 } else if ((value = option_value(&iArg, argc, argv, arg, "intercept"))) {
773 add_function_request(opts, STAT_INTERCEPT, value, "intercept");
774 } else if ((value = option_value(&iArg, argc, argv, arg, "exmmMean"))) {
775 add_simple_stat_request(opts, STAT_EXMM_MEAN, value);
776 } else if (option_name_matches(arg, "noOverwrite", NULL)) {
777 opts->overwrite = 0;
778 } else if (option_name_matches(arg, "overwrite", NULL)) {
779 opts->overwrite = 1;
780 } else if (option_name_matches(arg, "verbose", NULL)) {
781 opts->verbose = 1;
782 } else if (option_name_matches(arg, "nowarnings", NULL)) {
783 opts->noWarnings = 1;
784 } else if (option_name_matches(arg, "median", NULL) ||
785 option_name_matches(arg, "percentile", NULL) ||
786 option_name_matches(arg, "decilerange", NULL)) {
787 fprintf(stderr, "error: %s is not supported because it requires retaining page histories\n", arg);
788 exit(EXIT_FAILURE);
789 } else {
790 fprintf(stderr, "error: unknown option: %s\n", arg);
791 exit(EXIT_FAILURE);
792 }
793 }
794
795 if (opts->requests == 0) {
796 fprintf(stderr, "error: at least one statistic option is required\n");
797 exit(EXIT_FAILURE);
798 }
799 if (!is_directory(opts->inputDir)) {
800 fprintf(stderr, "error: -inputDir is not a directory: %s\n", opts->inputDir);
801 exit(EXIT_FAILURE);
802 }
803}

◆ path_exists()

static int path_exists ( const char * path)
static

Definition at line 358 of file sddsgroupedenvelope.c.

358 {
359 struct stat st;
360 return stat(path, &st) == 0;
361}

◆ process_directory()

static int process_directory ( const char * workDir,
OPTIONS * opts,
int requireInputs )
static

Definition at line 2061 of file sddsgroupedenvelope.c.

2061 {
2062 STRING_LIST files;
2063 GROUP_LIST groups;
2064 char *finalOutput;
2065 SDDS_DATASET templateInput;
2066 int templateOpen = 0;
2067 int status = PROCESS_ERROR;
2068
2069 memset(&files, 0, sizeof(files));
2070 memset(&groups, 0, sizeof(groups));
2071 finalOutput = xstrdup(opts->output);
2072
2073 collect_input_files(workDir, opts, finalOutput, &files);
2074 if (files.items == 0) {
2075 if (requireInputs)
2076 fprintf(stderr, "error: no input files matched patterns in %s\n", workDir);
2077 status = requireInputs ? PROCESS_ERROR : PROCESS_NO_INPUT;
2078 goto finish;
2079 }
2080
2081 if (path_exists(finalOutput)) {
2082 if (!opts->overwrite) {
2083 fprintf(stderr, "error: output file already exists: %s\n", finalOutput);
2084 goto finish;
2085 }
2086 if (remove(finalOutput) != 0) {
2087 fprintf(stderr, "error: unable to remove existing output file %s: %s\n", finalOutput, strerror(errno));
2088 goto finish;
2089 }
2090 }
2091
2092 if (opts->verbose) {
2093 long i;
2094 fprintf(stderr, "Using %ld input files in %s\n", files.items, workDir);
2095 for (i = 0; i < files.items; i++)
2096 fprintf(stderr, " %s\n", base_name(files.item[i]));
2097 }
2098
2099 if (!SDDS_InitializeInput(&templateInput, files.item[0])) {
2100 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2101 goto finish;
2102 }
2103 templateOpen = 1;
2104 if (!compile_stat_definitions(opts, &templateInput))
2105 goto finish;
2106
2107 if (!read_input_files(&files, opts, &groups))
2108 goto finish;
2109 warn_for_missing_requested_group_values(opts, &groups);
2110 if (groups.groups == 0) {
2111 fprintf(stderr, "error: no %s values selected in %s\n", opts->groupBy, workDir);
2112 goto finish;
2113 }
2114 if (!write_output_pages(finalOutput, &templateInput, opts, &groups))
2115 goto finish;
2116 status = PROCESS_DONE;
2117
2118finish:
2119 free_group_list(&groups, opts);
2120 if (templateOpen && !SDDS_Terminate(&templateInput))
2121 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2122 string_list_clear(&files);
2123 free(finalOutput);
2124 return status;
2125}
SDDS_InitializeInput
int32_t SDDS_InitializeInput(SDDS_DATASET *SDDS_dataset, char *filename)
Definition SDDS_input.c:50
SDDS_Terminate
int32_t SDDS_Terminate(SDDS_DATASET *SDDS_dataset)
Definition SDDS_input.c:1471
GROUP_LIST
Definition sddsgroupedenvelope.c:142
SDDS_DATASET
Definition SDDS.h:308

◆ read_input_file()

static int read_input_file ( const char * filename,
const OPTIONS * opts,
GROUP_LIST * groups )
static

Definition at line 1683 of file sddsgroupedenvelope.c.

1683 {
1684 SDDS_DATASET input;
1685 int pageCode;
1686 STRING_LIST seen;
1687
1688 memset(&seen, 0, sizeof(seen));
1689 if (opts->verbose)
1690 fprintf(stderr, "Reading %s\n", filename);
1691 if (!SDDS_InitializeInput(&input, (char *)filename)) {
1692 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1693 string_list_clear(&seen);
1694 return 0;
1695 }
1696
1697 while ((pageCode = SDDS_ReadPage(&input)) > 0) {
1698 char *groupName = NULL;
1699 ENVELOPE_GROUP *group;
1700 ROW_ACCUMULATOR *rowData;
1701 int64_t rows;
1702 int wantPage;
1703 long iStat;
1704
1705 groupName = SDDS_GetParameterAsString(&input, opts->groupBy, NULL);
1706 if (!groupName || !strlen(groupName)) {
1707 fprintf(stderr, "error: blank or missing %s in %s page %d\n", opts->groupBy, filename, pageCode);
1708 free(groupName);
1709 SDDS_Terminate(&input);
1710 string_list_clear(&seen);
1711 return 0;
1712 }
1713 if (string_list_contains(&seen, groupName)) {
1714 fprintf(stderr, "error: duplicate %s %s in %s\n", opts->groupBy, groupName, filename);
1715 free(groupName);
1716 SDDS_Terminate(&input);
1717 string_list_clear(&seen);
1718 return 0;
1719 }
1720 string_list_append(&seen, groupName);
1721
1722 wantPage = opts->groupValue.items == 0 || string_list_contains(&opts->groupValue, groupName);
1723 if (!wantPage) {
1724 free(groupName);
1725 continue;
1726 }
1727
1728 if (!(group = find_group(groups, groupName))) {
1729 void *groupValue = SDDS_GetParameter(&input, opts->groupBy, NULL);
1730 if (!groupValue) {
1731 fprintf(stderr, "error: unable to read grouping parameter %s\n", opts->groupBy);
1732 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1733 free(groupName);
1734 SDDS_Terminate(&input);
1735 string_list_clear(&seen);
1736 return 0;
1737 }
1738 group = add_group(groups, groupName, groupValue);
1739 }
1740 rows = SDDS_CountRowsOfInterest(&input);
1741 if (rows <= 0) {
1742 group->zeroRowPages++;
1743 free(groupName);
1744 continue;
1745 }
1746
1747 rowData = get_row_accumulator(group, opts, rows);
1748 for (iStat = 0; iStat < opts->stats; iStat++) {
1749 if (!accumulate_stat(&input, rowData, opts, iStat)) {
1750 free(groupName);
1751 SDDS_Terminate(&input);
1752 string_list_clear(&seen);
1753 return 0;
1754 }
1755 }
1756 rowData->pages++;
1757 free(groupName);
1758 }
1759
1760 if (pageCode == 0) {
1761 fprintf(stderr, "error: failed while reading pages from %s\n", filename);
1762 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1763 SDDS_Terminate(&input);
1764 string_list_clear(&seen);
1765 return 0;
1766 }
1767 if (!SDDS_Terminate(&input)) {
1768 fprintf(stderr, "error: failed to terminate input file %s\n", filename);
1769 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1770 string_list_clear(&seen);
1771 return 0;
1772 }
1773 string_list_clear(&seen);
1774 return 1;
1775}
SDDS_CountRowsOfInterest
int64_t SDDS_CountRowsOfInterest(SDDS_DATASET *SDDS_dataset)
Counts the number of rows marked as "of interest" in the current data table.
Definition SDDS_extract.c:364
SDDS_GetParameterAsString
char * SDDS_GetParameterAsString(SDDS_DATASET *SDDS_dataset, char *parameter_name, char **memory)
Retrieves the value of a specified parameter as a string from the current data table of an SDDS datas...
Definition SDDS_extract.c:2839
SDDS_GetParameter
void * SDDS_GetParameter(SDDS_DATASET *SDDS_dataset, char *parameter_name, void *memory)
Retrieves the value of a specified parameter from the current data table of a data set.
Definition SDDS_extract.c:2482
SDDS_ReadPage
int32_t SDDS_ReadPage(SDDS_DATASET *SDDS_dataset)
Definition SDDS_input.c:1007

◆ read_input_files()

static int read_input_files ( const STRING_LIST * files,
const OPTIONS * opts,
GROUP_LIST * groups )
static

Definition at line 1831 of file sddsgroupedenvelope.c.

1831 {
1832 if (opts->threads <= 1)
1833 return read_input_files_serial(files, opts, groups);
1834 return read_input_files_threaded(files, opts, groups);
1835}

◆ read_input_files_serial()

static int read_input_files_serial ( const STRING_LIST * files,
const OPTIONS * opts,
GROUP_LIST * groups )
static

Definition at line 1777 of file sddsgroupedenvelope.c.

1777 {
1778 long iFile;
1779
1780 for (iFile = 0; iFile < files->items; iFile++) {
1781 if (!read_input_file(files->item[iFile], opts, groups))
1782 return 0;
1783 }
1784 return 1;
1785}

◆ read_input_files_threaded()

static int read_input_files_threaded ( const STRING_LIST * files,
const OPTIONS * opts,
GROUP_LIST * groups )
static

Definition at line 1787 of file sddsgroupedenvelope.c.

1787 {
1788#if SDDSGROUPEDENVELOPE_USE_OPENMP
1789 FILE_READ_RESULT *result;
1790 long batchStart, iBatch, batchSize;
1791 int ok = 1;
1792 int threads = opts->threads;
1793
1794 if (threads > files->items)
1795 threads = (int)files->items;
1796 if (threads <= 1)
1797 return read_input_files_serial(files, opts, groups);
1798
1799 result = xcalloc((size_t)threads, sizeof(*result));
1800 omp_set_num_threads(threads);
1801
1802 for (batchStart = 0; ok && batchStart < files->items; batchStart += threads) {
1803 batchSize = files->items - batchStart;
1804 if (batchSize > threads)
1805 batchSize = threads;
1806
1807#pragma omp parallel for schedule(dynamic)
1808 for (iBatch = 0; iBatch < batchSize; iBatch++)
1809 result[iBatch].ok = read_input_file(files->item[batchStart + iBatch], opts, &result[iBatch].groups);
1810
1811 for (iBatch = 0; iBatch < batchSize; iBatch++) {
1812 if (!result[iBatch].ok)
1813 ok = 0;
1814 }
1815 if (ok) {
1816 for (iBatch = 0; iBatch < batchSize; iBatch++)
1817 merge_group_list(groups, &result[iBatch].groups, opts);
1818 }
1819 for (iBatch = 0; iBatch < batchSize; iBatch++) {
1820 free_group_list(&result[iBatch].groups, opts);
1821 memset(&result[iBatch], 0, sizeof(*result));
1822 }
1823 }
1824 free(result);
1825 return ok;
1826#else
1827 return read_input_files_serial(files, opts, groups);
1828#endif
1829}
FILE_READ_RESULT
Definition sddsgroupedenvelope.c:148

◆ read_parameter_double()

static int read_parameter_double ( SDDS_DATASET * input,
const char * name,
double * value )
static

Definition at line 1216 of file sddsgroupedenvelope.c.

1216 {
1217 if (!SDDS_GetParameterAsDouble(input, (char *)name, value)) {
1218 fprintf(stderr, "error: unable to read parameter %s as double\n", name);
1219 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
1220 return 0;
1221 }
1222 return 1;
1223}
SDDS_GetParameterAsDouble
double * SDDS_GetParameterAsDouble(SDDS_DATASET *SDDS_dataset, char *parameter_name, double *memory)
Retrieves the value of a specified parameter as a double from the current data table of an SDDS datas...
Definition SDDS_extract.c:2783

◆ result_name_exists()

static int result_name_exists ( const OPTIONS * opts,
const char * name )
static

Definition at line 901 of file sddsgroupedenvelope.c.

901 {
902 long i;
903 for (i = 0; i < opts->stats; i++) {
904 if (strcmp(opts->stat[i].resultColumn, name) == 0)
905 return 1;
906 }
907 return 0;
908}

◆ same_existing_file()

static int same_existing_file ( const char * path1,
const char * path2 )
static

Definition at line 363 of file sddsgroupedenvelope.c.

363 {
364#if defined(_WIN32)
365 BY_HANDLE_FILE_INFORMATION info1, info2;
366 HANDLE file1, file2;
367 int same = 0;
368
369 if (!path1 || !path2)
370 return 0;
371 if (_stricmp(path1, path2) == 0)
372 return 1;
373 file1 = CreateFileA(path1, 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
374 NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
375 if (file1 == INVALID_HANDLE_VALUE)
376 return 0;
377 file2 = CreateFileA(path2, 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
378 NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
379 if (file2 != INVALID_HANDLE_VALUE) {
380 if (GetFileInformationByHandle(file1, &info1) &&
381 GetFileInformationByHandle(file2, &info2)) {
382 same = info1.dwVolumeSerialNumber == info2.dwVolumeSerialNumber &&
383 info1.nFileIndexHigh == info2.nFileIndexHigh &&
384 info1.nFileIndexLow == info2.nFileIndexLow;
385 }
386 CloseHandle(file2);
387 }
388 CloseHandle(file1);
389 return same;
390#else
391 struct stat st1, st2;
392 if (!path1 || !path2)
393 return 0;
394 if (strcmp(path1, path2) == 0)
395 return 1;
396 if (stat(path1, &st1) != 0 || stat(path2, &st2) != 0)
397 return 0;
398 return st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino;
399#endif
400}

◆ setup_output_file()

static int setup_output_file ( SDDS_DATASET * output,
const char * outputFile,
SDDS_DATASET * input,
const OPTIONS * opts )
static

Definition at line 1657 of file sddsgroupedenvelope.c.

1657 {
1658 long iStat;
1659
1660 if (!SDDS_InitializeOutput(output, SDDS_BINARY, 0, NULL, "sddsgroupedenvelope output", (char *)outputFile)) {
1661 fprintf(stderr, "error: unable to initialize output file %s\n", outputFile);
1662 return 0;
1663 }
1664 output->layout.data_mode.column_major = opts->columnMajorOrder;
1665
1666 if (!SDDS_TransferParameterDefinition(output, input, opts->groupBy, opts->groupBy)) {
1667 fprintf(stderr, "error: unable to define grouping parameter %s in output\n", opts->groupBy);
1668 return 0;
1669 }
1670 for (iStat = 0; iStat < opts->stats; iStat++) {
1671 if (!define_stat_column(output, input, opts->stat + iStat)) {
1672 fprintf(stderr, "error: unable to define output column %s\n", opts->stat[iStat].resultColumn);
1673 return 0;
1674 }
1675 }
1676 if (!SDDS_WriteLayout(output)) {
1677 fprintf(stderr, "error: unable to write output layout for %s\n", outputFile);
1678 return 0;
1679 }
1680 return 1;
1681}
SDDS_InitializeOutput
int32_t SDDS_InitializeOutput(SDDS_DATASET *SDDS_dataset, int32_t data_mode, int32_t lines_per_row, const char *description, const char *contents, const char *filename)
Initializes the SDDS output dataset.
Definition SDDS_output.c:602
SDDS_WriteLayout
int32_t SDDS_WriteLayout(SDDS_DATASET *SDDS_dataset)
Writes the SDDS layout header to the output file.
Definition SDDS_output.c:900
SDDS_TransferParameterDefinition
int32_t SDDS_TransferParameterDefinition(SDDS_DATASET *target, SDDS_DATASET *source, char *name, char *newName)
Transfers a parameter definition from a source dataset to a target dataset.
Definition SDDS_transfer.c:73

◆ split_comma_list()

static void split_comma_list ( const char * value,
STRING_LIST * list )
static

Definition at line 288 of file sddsgroupedenvelope.c.

288 {
289 const char *start, *p;
290 start = value;
291 for (p = value; ; p++) {
292 if (*p == ',' || *p == 0) {
293 char *item = trimmed_copy(start, (size_t)(p - start));
294 if (!strlen(item)) {
295 fprintf(stderr, "error: empty item in option value '%s'\n", value);
296 free(item);
297 exit(EXIT_FAILURE);
298 }
299 string_list_append_owned(list, item);
300 if (*p == 0)
301 break;
302 start = p + 1;
303 }
304 }
305}

◆ stat_column_name()

static char * stat_column_name ( const STAT_DEFINITION * stat)
static

Definition at line 875 of file sddsgroupedenvelope.c.

875 {
876 char buffer[64];
877 const char *suffix = stat_suffix(stat->code);
878 const char *prefix = stat->sourceColumn;
879 size_t length;
880 char *name;
881
882 if (stat->code == STAT_COPY)
883 return xstrdup(stat->sourceColumn);
884 if (stat->code == STAT_SUM && stat->sumPower != 1) {
885 snprintf(buffer, sizeof(buffer), "%ld%s", stat->sumPower, suffix);
886 suffix = buffer;
887 }
888 if (stat->code == STAT_CMAXIMUM || stat->code == STAT_CMINIMUM ||
889 stat->code == STAT_PMAXIMUM || stat->code == STAT_PMINIMUM) {
890 length = strlen(stat->functionOf) + strlen(suffix) + strlen(stat->sourceColumn) + 1;
891 name = xmalloc(length);
892 snprintf(name, length, "%s%s%s", stat->functionOf, suffix, stat->sourceColumn);
893 return name;
894 }
895 length = strlen(prefix) + strlen(suffix) + 1;
896 name = xmalloc(length);
897 snprintf(name, length, "%s%s", prefix, suffix);
898 return name;
899}

◆ stat_suffix()

static const char * stat_suffix ( STAT_CODE code)
static

Definition at line 493 of file sddsgroupedenvelope.c.

493 {
494 switch (code) {
495 case STAT_COPY:
496 return "";
497 case STAT_MAXIMUM:
498 return "Max";
499 case STAT_MINIMUM:
500 return "Min";
501 case STAT_MEAN:
502 return "Mean";
503 case STAT_LARGEST:
504 return "Largest";
505 case STAT_SIGNED_LARGEST:
506 return "SignedLargest";
507 case STAT_SUM:
508 return "Sum";
509 case STAT_RMS:
510 return "Rms";
511 case STAT_STANDARD_DEVIATION:
512 return "StDev";
513 case STAT_SIGMA:
514 return "Sigma";
515 case STAT_WMEAN:
516 return "WMean";
517 case STAT_WSTANDARD_DEVIATION:
518 return "WStDev";
519 case STAT_WRMS:
520 return "WRms";
521 case STAT_WSIGMA:
522 return "WSigma";
523 case STAT_CMAXIMUM:
524 return "CMaximum";
525 case STAT_CMINIMUM:
526 return "CMinimum";
527 case STAT_PMAXIMUM:
528 return "PMaximum";
529 case STAT_PMINIMUM:
530 return "PMinimum";
531 case STAT_SLOPE:
532 return "Slope";
533 case STAT_INTERCEPT:
534 return "Intercept";
535 case STAT_EXMM_MEAN:
536 return "ExmmMean";
537 }
538 return "";
539}

◆ stat_uses_function_column()

static int stat_uses_function_column ( STAT_CODE code)
static

Definition at line 546 of file sddsgroupedenvelope.c.

546 {
547 return code == STAT_CMAXIMUM || code == STAT_CMINIMUM;
548}

◆ stat_uses_function_parameter()

static int stat_uses_function_parameter ( STAT_CODE code)
static

Definition at line 550 of file sddsgroupedenvelope.c.

550 {
551 return code == STAT_PMAXIMUM || code == STAT_PMINIMUM ||
552 code == STAT_SLOPE || code == STAT_INTERCEPT;
553}

◆ stat_uses_weight()

static int stat_uses_weight ( STAT_CODE code)
static

Definition at line 541 of file sddsgroupedenvelope.c.

541 {
542 return code == STAT_WMEAN || code == STAT_WSTANDARD_DEVIATION ||
543 code == STAT_WRMS || code == STAT_WSIGMA;
544}

◆ string_list_append()

static void string_list_append ( STRING_LIST * list,
const char * value )
static

Definition at line 252 of file sddsgroupedenvelope.c.

252 {
253 string_list_append_owned(list, xstrdup(value));
254}

◆ string_list_append_owned()

static void string_list_append_owned ( STRING_LIST * list,
char * value )
static

Definition at line 244 of file sddsgroupedenvelope.c.

244 {
245 if (list->items >= list->allocated) {
246 list->allocated = list->allocated ? 2 * list->allocated : 8;
247 list->item = xrealloc(list->item, sizeof(*list->item) * list->allocated);
248 }
249 list->item[list->items++] = value;
250}

◆ string_list_clear()

static void string_list_clear ( STRING_LIST * list)
static

Definition at line 256 of file sddsgroupedenvelope.c.

256 {
257 long i;
258 for (i = 0; i < list->items; i++)
259 free(list->item[i]);
260 free(list->item);
261 list->item = NULL;
262 list->items = list->allocated = 0;
263}

◆ string_list_contains()

static int string_list_contains ( const STRING_LIST * list,
const char * value )
static

Definition at line 265 of file sddsgroupedenvelope.c.

265 {
266 long i;
267 for (i = 0; i < list->items; i++) {
268 if (strcmp(list->item[i], value) == 0)
269 return 1;
270 }
271 return 0;
272}

◆ timestamp_seconds()

static int timestamp_seconds ( const char * path)
static

Definition at line 420 of file sddsgroupedenvelope.c.

420 {
421 const char *tail, *p;
422 tail = base_name(path);
423 for (p = tail; *p; p++) {
424 int h, m, s, n = 0;
425 if (!isdigit((unsigned char)*p))
426 continue;
427 if (sscanf(p, "%2d:%2d:%2d%n", &h, &m, &s, &n) == 3 && n >= 7 &&
428 h >= 0 && h < 24 && m >= 0 && m < 60 && s >= 0 && s < 60) {
429 return h * 3600 + m * 60 + s;
430 }
431 }
432 return -1;
433}

◆ trimmed_copy()

static char * trimmed_copy ( const char * start,
size_t length )
static

Definition at line 274 of file sddsgroupedenvelope.c.

274 {
275 const char *end = start + length;
276 char *copy;
277
278 while (start < end && isspace((unsigned char)*start))
279 start++;
280 while (end > start && isspace((unsigned char)*(end - 1)))
281 end--;
282 copy = xmalloc((size_t)(end - start) + 1);
283 memcpy(copy, start, (size_t)(end - start));
284 copy[end - start] = 0;
285 return copy;
286}

◆ warn_for_missing_requested_group_values()

static void warn_for_missing_requested_group_values ( const OPTIONS * opts,
const GROUP_LIST * groups )
static

Definition at line 2045 of file sddsgroupedenvelope.c.

2045 {
2046 long i;
2047 for (i = 0; i < opts->groupValue.items; i++) {
2048 long j;
2049 int found = 0;
2050 for (j = 0; j < groups->groups; j++) {
2051 if (strcmp(opts->groupValue.item[i], groups->group[j].name) == 0) {
2052 found = 1;
2053 break;
2054 }
2055 }
2056 if (!found)
2057 fprintf(stderr, "warning: requested %s value not found: %s\n", opts->groupBy, opts->groupValue.item[i]);
2058 }
2059}

◆ warn_zero_weight()

static void warn_zero_weight ( const OPTIONS * opts,
const STAT_DEFINITION * stat,
int64_t row )
static

Definition at line 1837 of file sddsgroupedenvelope.c.

1837 {
1838 if (!opts->noWarnings)
1839 fprintf(stderr, "warning: the total weight for row %" PRId64 " of %s is zero\n", row + 1, stat->sourceColumn);
1840}

◆ write_output_pages()

static int write_output_pages ( const char * outputFile,
SDDS_DATASET * templateInput,
const OPTIONS * opts,
GROUP_LIST * groups )
static

Definition at line 1970 of file sddsgroupedenvelope.c.

1970 {
1971 SDDS_DATASET output;
1972 long iGroup;
1973 int pagesWritten = 0;
1974
1975 if (!setup_output_file(&output, outputFile, templateInput, opts))
1976 goto error;
1977
1978 for (iGroup = 0; iGroup < groups->groups; iGroup++) {
1979 ENVELOPE_GROUP *group = groups->group + iGroup;
1980 ROW_ACCUMULATOR *rowData = best_row_accumulator(group);
1981 int64_t ignoredRows = 0;
1982 long iRowData, iStat;
1983
1984 if (!rowData) {
1985 fprintf(stderr, "error: all pages have zero rows for %s\n", group->name);
1986 goto error_with_output;
1987 }
1988 for (iRowData = 0; iRowData < group->rowDatas; iRowData++) {
1989 if (group->rowData + iRowData != rowData)
1990 ignoredRows += group->rowData[iRowData].pages;
1991 }
1992 if (opts->verbose && (group->zeroRowPages || ignoredRows)) {
1993 fprintf(stderr, "Row-count selection for %s: keeping %" PRId64 " pages with %" PRId64
1994 " rows; ignored %" PRId64 " zero-row pages and %" PRId64 " nonmatching pages\n",
1995 group->name, rowData->pages, rowData->rows, group->zeroRowPages, ignoredRows);
1996 }
1997
1998 if (!SDDS_StartPage(&output, rowData->rows) ||
1999 !SDDS_SetParameters(&output, SDDS_SET_BY_NAME | SDDS_PASS_BY_REFERENCE, opts->groupBy, group->value, NULL)) {
2000 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2001 goto error_with_output;
2002 }
2003
2004 for (iStat = 0; iStat < opts->stats; iStat++) {
2005 STAT_ACCUMULATOR *accumulator = rowData->stat + iStat;
2006 STAT_DEFINITION *stat = opts->stat + iStat;
2007 finalize_stat_values(rowData, opts, iStat);
2008 if (stat->code == STAT_COPY) {
2009 if (!SDDS_SetColumn(&output, SDDS_SET_BY_NAME, accumulator->copyData, rowData->rows, stat->resultColumn)) {
2010 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2011 goto error_with_output;
2012 }
2013 } else if (!SDDS_SetColumnFromDoubles(&output, SDDS_SET_BY_NAME, accumulator->value1, rowData->rows, stat->resultColumn)) {
2014 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2015 goto error_with_output;
2016 }
2017 }
2018
2019 if (!SDDS_WritePage(&output)) {
2020 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2021 goto error_with_output;
2022 }
2023 pagesWritten++;
2024 }
2025
2026 if (!SDDS_Terminate(&output)) {
2027 SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
2028 goto error;
2029 }
2030
2031 if (pagesWritten == 0) {
2032 fprintf(stderr, "error: no envelope pages were created\n");
2033 return 0;
2034 }
2035 if (opts->verbose)
2036 fprintf(stderr, "Wrote %s with %d pages\n", outputFile, pagesWritten);
2037 return 1;
2038
2039error_with_output:
2040 SDDS_Terminate(&output);
2041error:
2042 return 0;
2043}
SDDS_StartPage
int32_t SDDS_StartPage(SDDS_DATASET *SDDS_dataset, int64_t expected_n_rows)
Definition SDDS_dataprep.c:70
SDDS_SetParameters
int32_t SDDS_SetParameters(SDDS_DATASET *SDDS_dataset, int32_t mode,...)
Definition SDDS_dataprep.c:375
SDDS_SetColumnFromDoubles
int32_t SDDS_SetColumnFromDoubles(SDDS_DATASET *SDDS_dataset, int32_t mode, double *data, int64_t rows,...)
Sets the values for a single data column using double-precision floating-point numbers.
Definition SDDS_dataprep.c:1648
SDDS_SetColumn
int32_t SDDS_SetColumn(SDDS_DATASET *SDDS_dataset, int32_t mode, void *data, int64_t rows,...)
Sets the values for one data column in the current data table of an SDDS dataset.
Definition SDDS_dataprep.c:1542
SDDS_WritePage
int32_t SDDS_WritePage(SDDS_DATASET *SDDS_dataset)
Writes the current data table to the output file.
Definition SDDS_output.c:1229

◆ xcalloc()

static void * xcalloc ( size_t count,
size_t size )
static

Definition at line 209 of file sddsgroupedenvelope.c.

209 {
210 void *ptr;
211 if (count == 0 || size == 0) {
212 count = size = 1;
213 }
214 ptr = calloc(count, size);
215 if (!ptr) {
216 fprintf(stderr, "error: memory allocation failed\n");
217 exit(EXIT_FAILURE);
218 }
219 return ptr;
220}

◆ xmalloc()

static void * xmalloc ( size_t size)
static

Definition at line 197 of file sddsgroupedenvelope.c.

197 {
198 void *ptr;
199 if (size == 0)
200 size = 1;
201 ptr = malloc(size);
202 if (!ptr) {
203 fprintf(stderr, "error: memory allocation failed\n");
204 exit(EXIT_FAILURE);
205 }
206 return ptr;
207}

◆ xrealloc()

static void * xrealloc ( void * ptr,
size_t size )
static

Definition at line 222 of file sddsgroupedenvelope.c.

222 {
223 if (size == 0)
224 size = 1;
225 ptr = realloc(ptr, size);
226 if (!ptr) {
227 fprintf(stderr, "error: memory allocation failed\n");
228 exit(EXIT_FAILURE);
229 }
230 return ptr;
231}

◆ xstrdup()

static char * xstrdup ( const char * s)
static

Definition at line 233 of file sddsgroupedenvelope.c.

233 {
234 char *copy;
235 size_t len;
236 if (!s)
237 s = "";
238 len = strlen(s);
239 copy = xmalloc(len + 1);
240 memcpy(copy, s, len + 1);
241 return copy;
242}
Generated by doxygen 1.12.0