mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-12-21 08:39:46 +08:00
Code Issues Packages Projects Releases Wiki Activity
f5c9183dd5
netcdf-c/nc_test/ref_tst_http_nc4a.cdl

298 lines
17 KiB
Plaintext
Raw Normal View History

Provide byte-range reading of remote datasets re: issue https://github.com/Unidata/netcdf-c/issues/1251 Assume that you have the URL to a remote dataset which is a normal netcdf-3 or netcdf-4 file. This PR allows the netcdf-c to read that dataset's contents as a netcdf file using HTTP byte ranges if the remote server supports byte-range access. Originally, this PR was set up to access Amazon S3 objects, but it can also access other remote datasets such as those provided by a Thredds server via the HTTPServer access protocol. It may also work for other kinds of servers. Note that this is not intended as a true production capability because, as is known, this kind of access to can be quite slow. In addition, the byte-range IO drivers do not currently do any sort of optimization or caching. An additional goal here is to gain some experience with the Amazon S3 REST protocol. This architecture and its use documented in the file docs/byterange.dox. There are currently two test cases: 1. nc_test/tst_s3raw.c - this does a simple open, check format, close cycle for a remote netcdf-3 file and a remote netcdf-4 file. 2. nc_test/test_s3raw.sh - this uses ncdump to investigate some remote datasets. This PR also incorporates significantly changed model inference code (see the superceded PR https://github.com/Unidata/netcdf-c/pull/1259). 1. It centralizes the code that infers the dispatcher. 2. It adds support for byte-range URLs Other changes: 1. NC_HDF5_finalize was not being properly called by nc_finalize(). 2. Fix minor bug in ncgen3.l 3. fix memory leak in nc4info.c 4. add code to walk the .daprc triples and to replace protocol= fragment tag with a more general mode= tag. Final Note: Th inference code is still way too complicated. We need to move to the validfile() model used by netcdf Java, where each dispatcher is asked if it can process the file. This decentralizes the inference code. This will be done after all the major new dispatchers (PIO, Zarr, etc) have been implemented.
2019-01-02 09:27:36 +08:00
netcdf OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316 {
dimensions:
y = 1500 ;
x = 2500 ;
number_of_time_bounds = 2 ;
band = 1 ;
number_of_image_bounds = 2 ;
num_star_looks = 24 ;
variables:
short Rad(y, x) ;
Rad:_FillValue = 4095s ;
Rad:long_name = "ABI L1b Radiances" ;
Rad:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
Rad:_Unsigned = "true" ;
Rad:sensor_band_bit_depth = 12b ;
Rad:valid_range = 0s, 4094s ;
Rad:scale_factor = 0.04572892f ;
Rad:add_offset = -1.6443f ;
Rad:units = "mW m-2 sr-1 (cm-1)-1" ;
Rad:resolution = "y: 0.000056 rad x: 0.000056 rad" ;
Rad:coordinates = "band_id band_wavelength t y x" ;
Rad:grid_mapping = "goes_imager_projection" ;
Rad:cell_methods = "t: point area: point" ;
Rad:ancillary_variables = "DQF" ;
byte DQF(y, x) ;
DQF:_FillValue = -1b ;
DQF:long_name = "ABI L1b Radiances data quality flags" ;
DQF:standard_name = "status_flag" ;
DQF:_Unsigned = "true" ;
DQF:valid_range = 0b, 3b ;
DQF:units = "1" ;
DQF:coordinates = "band_id band_wavelength t y x" ;
DQF:grid_mapping = "goes_imager_projection" ;
DQF:cell_methods = "t: point area: point" ;
DQF:flag_values = 0b, 1b, 2b, 3b ;
DQF:flag_meanings = "good_pixel_qf conditionally_usable_pixel_qf out_of_range_pixel_qf no_value_pixel_qf" ;
DQF:number_of_qf_values = 4b ;
DQF:percent_good_pixel_qf = 1.f ;
DQF:percent_conditionally_usable_pixel_qf = 0.f ;
DQF:percent_out_of_range_pixel_qf = 0.f ;
DQF:percent_no_value_pixel_qf = 0.f ;
double t ;
t:long_name = "J2000 epoch mid-point between the start and end image scan in seconds" ;
t:standard_name = "time" ;
t:units = "seconds since 2000-01-01 12:00:00" ;
t:axis = "T" ;
t:bounds = "time_bounds" ;
short y(y) ;
y:scale_factor = -5.6e-05f ;
y:add_offset = 0.126532f ;
y:units = "rad" ;
y:axis = "Y" ;
y:long_name = "GOES fixed grid projection y-coordinate" ;
y:standard_name = "projection_y_coordinate" ;
short x(x) ;
x:scale_factor = 5.6e-05f ;
x:add_offset = -0.075012f ;
x:units = "rad" ;
x:axis = "X" ;
x:long_name = "GOES fixed grid projection x-coordinate" ;
x:standard_name = "projection_x_coordinate" ;
double time_bounds(number_of_time_bounds) ;
time_bounds:long_name = "Scan start and end times in seconds since epoch (2000-01-01 12:00:00)" ;
int goes_imager_projection ;
goes_imager_projection:long_name = "GOES-R ABI fixed grid projection" ;
goes_imager_projection:grid_mapping_name = "geostationary" ;
goes_imager_projection:perspective_point_height = 35786023. ;
goes_imager_projection:semi_major_axis = 6378137. ;
goes_imager_projection:semi_minor_axis = 6356752.31414 ;
goes_imager_projection:inverse_flattening = 298.2572221 ;
goes_imager_projection:latitude_of_projection_origin = 0. ;
goes_imager_projection:longitude_of_projection_origin = -89.5 ;
goes_imager_projection:sweep_angle_axis = "x" ;
float y_image ;
y_image:long_name = "GOES-R fixed grid projection y-coordinate center of image" ;
y_image:standard_name = "projection_y_coordinate" ;
y_image:units = "rad" ;
y_image:axis = "Y" ;
float y_image_bounds(number_of_image_bounds) ;
y_image_bounds:long_name = "GOES-R fixed grid projection y-coordinate north/south extent of image" ;
float x_image ;
x_image:long_name = "GOES-R fixed grid projection x-coordinate center of image" ;
x_image:standard_name = "projection_x_coordinate" ;
x_image:units = "rad" ;
x_image:axis = "X" ;
float x_image_bounds(number_of_image_bounds) ;
x_image_bounds:long_name = "GOES-R fixed grid projection x-coordinate west/east extent of image" ;
float nominal_satellite_subpoint_lat ;
nominal_satellite_subpoint_lat:long_name = "nominal satellite subpoint latitude (platform latitude)" ;
nominal_satellite_subpoint_lat:standard_name = "latitude" ;
nominal_satellite_subpoint_lat:_FillValue = -999.f ;
nominal_satellite_subpoint_lat:units = "degrees_north" ;
float nominal_satellite_subpoint_lon ;
nominal_satellite_subpoint_lon:long_name = "nominal satellite subpoint longitude (platform longitude)" ;
nominal_satellite_subpoint_lon:standard_name = "longitude" ;
nominal_satellite_subpoint_lon:_FillValue = -999.f ;
nominal_satellite_subpoint_lon:units = "degrees_east" ;
float nominal_satellite_height ;
nominal_satellite_height:long_name = "nominal satellite height above GRS 80 ellipsoid (platform altitude)" ;
nominal_satellite_height:standard_name = "height_above_reference_ellipsoid" ;
nominal_satellite_height:_FillValue = -999.f ;
nominal_satellite_height:units = "km" ;
float geospatial_lat_lon_extent ;
geospatial_lat_lon_extent:long_name = "geospatial latitude and longitude references" ;
geospatial_lat_lon_extent:geospatial_westbound_longitude = -140.6163f ;
geospatial_lat_lon_extent:geospatial_northbound_latitude = 52.76771f ;
geospatial_lat_lon_extent:geospatial_eastbound_longitude = -49.17929f ;
geospatial_lat_lon_extent:geospatial_southbound_latitude = 14.00016f ;
geospatial_lat_lon_extent:geospatial_lat_center = 29.294f ;
geospatial_lat_lon_extent:geospatial_lon_center = -91.406f ;
geospatial_lat_lon_extent:geospatial_lat_nadir = 0.f ;
geospatial_lat_lon_extent:geospatial_lon_nadir = -89.5f ;
geospatial_lat_lon_extent:geospatial_lat_units = "degrees_north" ;
geospatial_lat_lon_extent:geospatial_lon_units = "degrees_east" ;
byte yaw_flip_flag ;
yaw_flip_flag:long_name = "Flag indicating the spacecraft is operating in yaw flip configuration" ;
yaw_flip_flag:_Unsigned = "true" ;
yaw_flip_flag:_FillValue = -1b ;
yaw_flip_flag:valid_range = 0b, 1b ;
yaw_flip_flag:units = "1" ;
yaw_flip_flag:coordinates = "t" ;
yaw_flip_flag:flag_values = "0 1" ;
yaw_flip_flag:flag_meanings = "false true" ;
byte band_id(band) ;
band_id:long_name = "ABI band number" ;
band_id:standard_name = "sensor_band_identifier" ;
band_id:units = "1" ;
float band_wavelength(band) ;
band_wavelength:long_name = "ABI band central wavelength" ;
band_wavelength:standard_name = "sensor_band_central_radiation_wavelength" ;
band_wavelength:units = "um" ;
float esun ;
esun:long_name = "bandpass-weighted solar irradiance at the mean Earth-Sun distance" ;
esun:standard_name = "toa_shortwave_irradiance_per_unit_wavelength" ;
esun:_FillValue = -999.f ;
esun:units = "W m-2 um-1" ;
esun:coordinates = "band_id band_wavelength t" ;
esun:cell_methods = "t: mean" ;
float kappa0 ;
kappa0:long_name = "Inverse of the incoming top of atmosphere radiance at current earth-sun distance (PI d2 esun-1)-1, where d is the ratio of instantaneous Earth-Sun distance divided by the mean Earth-Sun distance, esun is the bandpass-weighted solar irradiance and PI is a standard constant used to convert ABI L1b radiance to reflectance" ;
kappa0:_FillValue = -999.f ;
kappa0:units = "(W m-2 um-1)-1" ;
kappa0:coordinates = "band_id band_wavelength t" ;
kappa0:cell_methods = "t: mean" ;
float planck_fk1 ;
planck_fk1:long_name = "wavenumber-dependent coefficient (2 h c2/ nu3) used in the ABI emissive band monochromatic brightness temperature computation, where nu =central wavenumber and h and c are standard constants" ;
planck_fk1:_FillValue = -999.f ;
planck_fk1:units = "W m-1" ;
planck_fk1:coordinates = "band_id band_wavelength" ;
float planck_fk2 ;
planck_fk2:long_name = "wavenumber-dependent coefficient (h c nu/b) used in the ABI emissive band monochromatic brightness temperature computation, where nu = central wavenumber and h, c, and b are standard constants" ;
planck_fk2:_FillValue = -999.f ;
planck_fk2:units = "K" ;
planck_fk2:coordinates = "band_id band_wavelength" ;
float planck_bc1 ;
planck_bc1:long_name = "spectral bandpass correction offset for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
planck_bc1:_FillValue = -999.f ;
planck_bc1:units = "K" ;
planck_bc1:coordinates = "band_id band_wavelength" ;
float planck_bc2 ;
planck_bc2:long_name = "spectral bandpass correction scale factor for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ;
planck_bc2:_FillValue = -999.f ;
planck_bc2:units = "1" ;
planck_bc2:coordinates = "band_id band_wavelength" ;
int valid_pixel_count ;
valid_pixel_count:long_name = "number of good and conditionally usable pixels" ;
valid_pixel_count:_FillValue = -1 ;
valid_pixel_count:units = "count" ;
valid_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
valid_pixel_count:grid_mapping = "goes_imager_projection" ;
valid_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
int missing_pixel_count ;
missing_pixel_count:long_name = "number of missing pixels" ;
missing_pixel_count:_FillValue = -1 ;
missing_pixel_count:units = "count" ;
missing_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
missing_pixel_count:grid_mapping = "goes_imager_projection" ;
missing_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: missing ABI fixed grid pixels only)" ;
int saturated_pixel_count ;
saturated_pixel_count:long_name = "number of saturated pixels" ;
saturated_pixel_count:_FillValue = -1 ;
saturated_pixel_count:units = "count" ;
saturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
saturated_pixel_count:grid_mapping = "goes_imager_projection" ;
saturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: radiometrically saturated geolocated/not missing pixels only)" ;
int undersaturated_pixel_count ;
undersaturated_pixel_count:long_name = "number of undersaturated pixels" ;
undersaturated_pixel_count:_FillValue = -1 ;
undersaturated_pixel_count:units = "count" ;
undersaturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ;
undersaturated_pixel_count:grid_mapping = "goes_imager_projection" ;
undersaturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000056 rad comment: radiometrically undersaturated geolocated/not missing pixels only)" ;
float min_radiance_value_of_valid_pixels ;
min_radiance_value_of_valid_pixels:long_name = "minimum radiance value of pixels" ;
min_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
min_radiance_value_of_valid_pixels:_FillValue = -999.f ;
min_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
min_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
min_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
min_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
min_radiance_value_of_valid_pixels:cell_methods = "t: sum area: minimum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float max_radiance_value_of_valid_pixels ;
max_radiance_value_of_valid_pixels:long_name = "maximum radiance value of pixels" ;
max_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
max_radiance_value_of_valid_pixels:_FillValue = -999.f ;
max_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
max_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
max_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
max_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
max_radiance_value_of_valid_pixels:cell_methods = "t: sum area: maximum (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float mean_radiance_value_of_valid_pixels ;
mean_radiance_value_of_valid_pixels:long_name = "mean radiance value of pixels" ;
mean_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
mean_radiance_value_of_valid_pixels:_FillValue = -999.f ;
mean_radiance_value_of_valid_pixels:valid_range = -1.6443f, 185.5699f ;
mean_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
mean_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
mean_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
mean_radiance_value_of_valid_pixels:cell_methods = "t: sum area: mean (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float std_dev_radiance_value_of_valid_pixels ;
std_dev_radiance_value_of_valid_pixels:long_name = "standard deviation of radiance values of pixels" ;
std_dev_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavenumber" ;
std_dev_radiance_value_of_valid_pixels:_FillValue = -999.f ;
std_dev_radiance_value_of_valid_pixels:units = "mW m-2 sr-1 (cm-1)-1" ;
std_dev_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ;
std_dev_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ;
std_dev_radiance_value_of_valid_pixels:cell_methods = "t: sum area: standard_deviation (interval: 0.000056 rad comment: good and conditionally usable quality pixels only)" ;
float percent_uncorrectable_L0_errors ;
percent_uncorrectable_L0_errors:long_name = "percent data lost due to uncorrectable L0 errors" ;
percent_uncorrectable_L0_errors:_FillValue = -999.f ;
percent_uncorrectable_L0_errors:valid_range = 0.f, 1.f ;
percent_uncorrectable_L0_errors:units = "percent" ;
percent_uncorrectable_L0_errors:coordinates = "t y_image x_image" ;
percent_uncorrectable_L0_errors:grid_mapping = "goes_imager_projection" ;
percent_uncorrectable_L0_errors:cell_methods = "t: sum area: sum (uncorrectable L0 errors only)" ;
float earth_sun_distance_anomaly_in_AU ;
earth_sun_distance_anomaly_in_AU:long_name = "earth sun distance anomaly in astronomical units" ;
earth_sun_distance_anomaly_in_AU:_FillValue = -999.f ;
earth_sun_distance_anomaly_in_AU:units = "ua" ;
earth_sun_distance_anomaly_in_AU:coordinates = "t" ;
earth_sun_distance_anomaly_in_AU:cell_methods = "t: mean" ;
int algorithm_dynamic_input_data_container ;
algorithm_dynamic_input_data_container:long_name = "container for filenames of dynamic algorithm input data" ;
algorithm_dynamic_input_data_container:input_ABI_L0_data = "OR_ABI-L0-C-M3_G16_s20170590337505_e20170590340289_c*.nc" ;
int processing_parm_version_container ;
processing_parm_version_container:long_name = "container for processing parameter filenames" ;
processing_parm_version_container:L1b_processing_parm_version = "ABI-L1b-PARM_G16_v01r00.zip" ;
int algorithm_product_version_container ;
algorithm_product_version_container:long_name = "container for algorithm package filename and product version" ;
algorithm_product_version_container:algorithm_version = "OR_ABI-L1b-ALG-COMMON_v01r00.zip" ;
algorithm_product_version_container:product_version = "v01r00" ;
double t_star_look(num_star_looks) ;
t_star_look:long_name = "J2000 epoch time of star observed in seconds" ;
t_star_look:standard_name = "time" ;
t_star_look:units = "seconds since 2000-01-01 12:00:00" ;
t_star_look:axis = "T" ;
float band_wavelength_star_look(num_star_looks) ;
band_wavelength_star_look:long_name = "ABI band central wavelength associated with observed star" ;
band_wavelength_star_look:standard_name = "sensor_band_central_radiation_wavelength" ;
band_wavelength_star_look:units = "um" ;
short star_id(num_star_looks) ;
star_id:long_name = "ABI star catalog identifier associated with observed star" ;
star_id:_Unsigned = "true" ;
star_id:_FillValue = -1s ;
star_id:coordinates = "band_id band_wavelength_star_look t_star_look" ;
// global attributes:
:naming_authority = "gov.nesdis.noaa" ;
:Conventions = "CF-1.7" ;
:Metadata_Conventions = "Unidata Dataset Discovery v1.0" ;
:standard_name_vocabulary = "CF Standard Name Table (v25, 05 July 2013)" ;
:institution = "DOC/NOAA/NESDIS > U.S. Department of Commerce, National Oceanic and Atmospheric Administration, National Environmental Satellite, Data, and Information Services" ;
:project = "GOES" ;
:production_site = "WCDAS" ;
:production_environment = "OE" ;
:spatial_resolution = "2km at nadir" ;
:orbital_slot = "GOES-Test" ;
:platform_ID = "G16" ;
:instrument_type = "GOES R Series Advanced Baseline Imager" ;
:scene_id = "CONUS" ;
:instrument_ID = "FM1" ;
:title = "ABI L1b Radiances" ;
:summary = "Single emissive band ABI L1b Radiance Products are digital maps of outgoing radiance values at the top of the atmosphere for IR bands." ;
:keywords = "SPECTRAL/ENGINEERING > INFRARED WAVELENGTHS > INFRARED RADIANCE" ;
:keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Earth Science Keywords, Version 7.0.0.0.0" ;
:iso_series_metadata_id = "a70be540-c38b-11e0-962b-0800200c9a66" ;
:license = "Unclassified data. Access is restricted to approved users only." ;
:processing_level = "L1b" ;
:cdm_data_type = "Image" ;
:dataset_name = "OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc" ;
:production_data_source = "Realtime" ;
:timeline_id = "ABI Mode 3" ;
:date_created = "2017-02-28T03:40:31.6Z" ;
:time_coverage_start = "2017-02-28T03:37:50.5Z" ;
:time_coverage_end = "2017-02-28T03:40:28.9Z" ;
:id = "037d7688-bd22-4ffb-9727-45e0c2890a72" ;
}
Reference in New Issue Copy Permalink