mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-27 07:30:33 +08:00
Code Issues Packages Projects Releases Wiki Activity
finalize.tmp
netcdf-c/nc_test/quick_large_files.c
Ward Fisher 763750fab5 More copyright stanza updates.
2018-12-06 14:56:42 -07:00

574 lines
22 KiB
C
Raw Permalink Blame History

/*
Copyright 2004-2018, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
This program (quickly, but not thoroughly) tests the large file
features. It turns off fill mode to quickly create an 8 gb file, and
write one value is written, nothing is read.
Ed Hartnett
*/
#include <config.h>
#include <nc_tests.h>
#include "err_macros.h"
#include <netcdf.h>
#include <stdio.h>
#include <string.h>
#define NUMDIMS 1
#define NUMVARS 2
/* This is the magic number for classic format limits: 2 GiB - 4
bytes. */
#define MAX_CLASSIC_BYTES 2147483644
/* This is the magic number for 64-bit offset format limits: 4 GiB - 4
bytes. */
#define MAX_64OFFSET_BYTES 4294967292
/* Handy for constucting tests. */
#define QTR_CLASSIC_MAX (MAX_CLASSIC_BYTES/4)
/* We will create this file. */
#define FILE_NAME "quick_large_files.nc"
int
main(int argc, char **argv)
{
int ncid, spockid, kirkid, dimids[NUMDIMS], recdimid, scottyid;
int int_val_in, int_val_out = 99;
double double_val_in, double_val_out = 1.79769313486230e+308; /* from ncx.h */
size_t index[2] = {QTR_CLASSIC_MAX-1, 0};
char file_name[NC_MAX_NAME + 1];
/* These are for the revolutionary generals tests. */
int cromwellid, collinsid, washingtonid;
int napoleanid, dimids_gen[4], dimids_gen1[4];
/* All create modes will be anded to this. All tests will be run
twice, with and without NC_SHARE.*/
int cmode_run;
int cflag = NC_CLOBBER;
int res;
printf("\n*** Testing large files, quickly.\n");
for (cmode_run=0; cmode_run<2; cmode_run++)
{
size_t big_index = (size_t)MAX_CLASSIC_BYTES + 10;
/* On second pass, try using NC_SHARE. */
if (cmode_run == 1)
{
cflag |= NC_SHARE;
printf("*** Turned on NC_SHARE for subsequent tests.\n");
}
/* Create a netCDF 64-bit offset format file. Write a value. */
sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
printf("*** Creating %s for 64-bit offset large file test...", file_name);
{
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "longdim", QTR_CLASSIC_MAX, dimids)))
ERR;
/* test bug fix writing record beyond 2**31 */
if ((res = nc_def_dim(ncid, "longerdim", NC_UNLIMITED, &recdimid)))
ERR;
if ((res = nc_def_var(ncid, "spock", NC_DOUBLE, NUMDIMS,
dimids, &spockid)))
ERR;
if ((res = nc_def_var(ncid, "kirk", NC_DOUBLE, NUMDIMS,
dimids, &kirkid)))
ERR;
if ((res = nc_def_var(ncid, "scotty", NC_BYTE, 1,
&recdimid, &scottyid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
if ((res = nc_put_var1_double(ncid, kirkid, index, &double_val_out)))
ERR;
if ((res = nc_put_var1_int(ncid, scottyid, &big_index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, scottyid, &big_index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
ERR;
if ((res = nc_close(ncid)))
ERR;
}
SUMMARIZE_ERR;
/* How about a meteorological data file about the weather
experience by various generals of revolutionary armies?
This has 3 dims, 4 vars. The dimensions are such that this will
(just barely) not fit in a classic format file. The first three
vars are cromwell, 536870911 bytes, washington, 2*536870911
bytes, and napolean, 536870911 bytes. That's a grand total of
2147483644 bytes. Recall our magic limit for the combined size
of all fixed vars: 2 GiB - 4 bytes, or 2147483644. So you would
think these would exactly fit, unless you realized that
everything is rounded to a 4 byte boundary, so you need to add
some bytes for that (how many?), and that pushes us over the
limit.
We will create this file twice, once to ensure it succeeds (with
64-bit offset format), and once to make sure it fails (with
classic format). Then some variations to check record var
boundaries.
*/
printf("*** Now a 64-bit offset, large file, fixed var test...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
QTR_CLASSIC_MAX, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
QTR_CLASSIC_MAX, &dimids_gen[1])))
ERR;
if ((res = nc_def_dim(ncid, "ruthlessness", 100, &dimids_gen[2])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_BYTE, 1, &dimids_gen[0],
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 1, &dimids_gen[1],
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 1, &dimids_gen[0],
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, &dimids_gen[2],
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
SUMMARIZE_ERR;
/* Write a value or two just for fun. */
/*index[0] = QTR_CLASSIC_MAX - 296;
if ((res = nc_put_var1_int(ncid, napoleanid, index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, napoleanid, index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
BAIL2;*/
printf("*** Now writing some values...");
index[0] = QTR_CLASSIC_MAX - 295;
if ((res = nc_put_var1_int(ncid, napoleanid, index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, napoleanid, index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
ERR;
index[0] = QTR_CLASSIC_MAX - 1;
if ((res = nc_put_var1_int(ncid, napoleanid, index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, napoleanid, index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
ERR;
index[0] = QTR_CLASSIC_MAX - 1;
if ((res = nc_put_var1_int(ncid, washingtonid, index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, washingtonid, index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* This time it should fail, because we're trying to cram this into
a classic format file. nc_enddef will detect our violations and
give an error. We've*/
printf("*** Now a classic file which will fail...");
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
QTR_CLASSIC_MAX, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
QTR_CLASSIC_MAX, &dimids_gen[1])))
ERR;
if ((res = nc_def_dim(ncid, "ruthlessness", 100, &dimids_gen[2])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_BYTE, 1, &dimids_gen[0],
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 1, &dimids_gen[1],
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 1, &dimids_gen[0],
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, &dimids_gen[2],
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)) != NC_EVARSIZE)
ERR;
if ((res = nc_close(ncid)) != NC_EVARSIZE)
ERR;
SUMMARIZE_ERR;
/* This will create some max sized 64-bit offset format fixed vars. */
printf("*** Now a 64-bit offset, simple fixed var create test...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[0])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_SHORT, 1, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_SHORT, 1, dimids_gen,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, dimids_gen,
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* This will exceed the 64-bit offset format limits for one of the
fixed vars. */
printf("*** Now a 64-bit offset, over-sized file that will fail...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
/* max dim size is MAX_CLASSIC_BYTES. */
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, dimids_gen)))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_DOUBLE, 1, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 1, dimids_gen,
&washingtonid)))
if ((res = nc_enddef(ncid)) != NC_EVARSIZE)
ERR;
if ((res = nc_close(ncid)) != NC_EVARSIZE)
ERR;
SUMMARIZE_ERR;
/* Now let's see about record vars. First create a 64-bit offset
file with three rec variables, each with the same numbers as
defined above for the fixed var tests. This should all work. */
printf("*** Now a 64-bit offset, record var file...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
QTR_CLASSIC_MAX, &dimids_gen[1])))
ERR;
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
QTR_CLASSIC_MAX, &dimids_gen[2])))
ERR;
if ((res = nc_def_dim(ncid, "ruthlessness", 100, &dimids_gen[3])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_BYTE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 2, dimids_gen,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, &dimids_gen[2],
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* Now try this record file in classic format. It should fail and
the enddef. Too many bytes in the first record.*/
printf("*** Now a classic file that's too big and will fail...");
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
QTR_CLASSIC_MAX, &dimids_gen[1])))
ERR;
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
QTR_CLASSIC_MAX, &dimids_gen[2])))
ERR;
if ((res = nc_def_dim(ncid, "ruthlessness", 100, &dimids_gen[3])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_BYTE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 2, dimids_gen,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, &dimids_gen[2],
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)) != NC_EVARSIZE)
ERR;
if ((res = nc_close(ncid)) != NC_EVARSIZE)
ERR;
SUMMARIZE_ERR;
/* Now try this record file in classic format. It just barely
passes at the enddef. Almost, but not quite, too many bytes in
the first record. Since I'm adding a fixed variable (Collins),
I don't get the last record size exemption. */
printf("*** Now a classic file with recs and one fixed will fail...");
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
if ((res = nc_def_dim(ncid, "ruthlessness", 100, &dimids_gen[2])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_BYTE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 1, &dimids_gen[2],
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* Try a classic file with several records, and the last record var
with a record size greater than our magic number of 2 GiB - 4
bytes. We'll start with just one oversized record var. This
should work. Cromwell has been changed to NC_DOUBLE, and that
increases his size to 2147483644 (the max dimension size) times
8, or about 16 GB per record. Zowie! (Mind you, Cromwell
certainly had a great deal of revolutionary fervor.)
*/
printf("*** Now a classic file with one large rec var...");
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_DOUBLE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
index[0] = 0;
index[1] = MAX_CLASSIC_BYTES - 1;
if ((res = nc_put_var1_double(ncid, cromwellid, index, &double_val_out)))
ERR;
if ((res = nc_get_var1_double(ncid, cromwellid, index, &double_val_in)))
ERR;
if (double_val_in != double_val_out)
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* This is a classic format file with an extra-large last record
var. */
printf("*** Now a classic file with extra-large last record var...") ;
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
dimids_gen1[0] = dimids_gen[0];
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
5368, &dimids_gen1[1])))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen1,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 2, dimids_gen1,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 2, dimids_gen1,
&collinsid)))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_DOUBLE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
index[0] = 0;
index[1] = MAX_CLASSIC_BYTES - 1;
if ((res = nc_put_var1_double(ncid, cromwellid, index, &double_val_out)))
ERR;
if ((res = nc_get_var1_double(ncid, cromwellid, index, &double_val_in)))
ERR;
if (double_val_in != double_val_out)
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
/* This is a classic format file with an extra-large second to last
record var. But this time it won't work, because the size
exemption only applies to the last record var. Note that one
dimension is small (5000). */
printf("*** Now a classic file xtra-large 2nd to last var that will fail...");
if ((res = nc_create(file_name, cflag, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
dimids_gen1[0] = dimids_gen[0];
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
5000, &dimids_gen1[1])))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen1,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 2, dimids_gen1,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_DOUBLE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 2, dimids_gen1,
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)) != NC_EVARSIZE)
ERR;
if ((res = nc_close(ncid)) != NC_EVARSIZE)
ERR;
SUMMARIZE_ERR;
/* Now try an extra large second to last ver with 64-bit
offset. This won't work either, because the cromwell var is so
large. It exceeds the 4GiB - 4 byte per record limit for record
vars. */
printf("*** Now a 64-bit offset file with too-large rec var that will fail...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
dimids_gen1[0] = dimids_gen[0];
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
5368, &dimids_gen1[1])))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen1,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_BYTE, 2, dimids_gen1,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_DOUBLE, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 2, dimids_gen1,
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)) != NC_EVARSIZE)
ERR;
if ((res = nc_close(ncid)) != NC_EVARSIZE)
ERR;
SUMMARIZE_ERR;
/* A 64-bit offset record file that just fits... */
printf("*** Now a 64 bit-offset file that just fits...");
if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid)))
ERR;
if ((res = nc_set_fill(ncid, NC_NOFILL, NULL)))
ERR;
if ((res = nc_def_dim(ncid, "political_trouble",
NC_UNLIMITED, &dimids_gen[0])))
ERR;
if ((res = nc_def_dim(ncid, "revolutionary_fervor",
MAX_CLASSIC_BYTES, &dimids_gen[1])))
ERR;
dimids_gen1[0] = dimids_gen[0];
if ((res = nc_def_dim(ncid, "post_revoultionary_hangover",
MAX_CLASSIC_BYTES, &dimids_gen1[1])))
ERR;
if ((res = nc_def_var(ncid, "Washington", NC_SHORT, 2, dimids_gen1,
&washingtonid)))
ERR;
if ((res = nc_def_var(ncid, "Napolean", NC_SHORT, 2, dimids_gen1,
&napoleanid)))
ERR;
if ((res = nc_def_var(ncid, "Cromwell", NC_SHORT, 2, dimids_gen,
&cromwellid)))
ERR;
if ((res = nc_def_var(ncid, "Collins", NC_DOUBLE, 2, dimids_gen1,
&collinsid)))
ERR;
if ((res = nc_enddef(ncid)))
ERR;
index[0] = 0;
index[1] = MAX_CLASSIC_BYTES - 1;
if ((res = nc_put_var1_int(ncid, cromwellid, index, &int_val_out)))
ERR;
if ((res = nc_get_var1_int(ncid, cromwellid, index, &int_val_in)))
ERR;
if (int_val_in != int_val_out)
ERR;
if ((res = nc_close(ncid)))
ERR;
SUMMARIZE_ERR;
} /* end of cmode run */
/* Delete the huge data file we created. */
(void) remove(file_name);
FINAL_RESULTS;
return 0;
}
Reference in New Issue View Git Blame Copy Permalink