NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.
To learn about what's going on, see About the Archive Site.
NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.
To learn about what's going on, see About the Archive Site.
Sure. The rest is the same. I'll do binary search next.
First:
netcdf a {
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
depth:_Storage = "chunked" ;
depth:_ChunkSizes = 2 ;
depth:_DeflateLevel = 9 ;
double lat(lat) ;
lat:long_name = "Latitude" ;
lat:standard_name = "latitude" ;
lat:units = "degrees_north" ;
lat:point_spacing = "even" ;
lat:axis = "Y" ;
lat:NAVO_code = 1 ;
lat:_Storage = "chunked" ;
lat:_ChunkSizes = 2001 ;
lat:_DeflateLevel = 9 ;
double lon(lon) ;
lon:long_name = "Longitude" ;
lon:standard_name = "longitude" ;
lon:units = "degrees_east" ;
lon:modulo = "360 degrees" ;
lon:axis = "X" ;
lon:NAVO_code = 2 ;
lon:_Storage = "chunked" ;
lon:_ChunkSizes = 4500 ;
lon:_DeflateLevel = 9 ;
double time(time) ;
time:long_name = "Valid Time" ;
time:units = "hours since 2000-01-01 00:00:00" ;
time:time_origin = "2000-01-01 00:00:00" ;
time:calendar = "gregorian" ;
time:axis = "T" ;
time:NAVO_code = 13 ;
time:_Storage = "chunked" ;
time:_ChunkSizes = 1 ;
time:_DeflateLevel = 9 ;
short water_u(time, depth, lat, lon) ;
water_u:long_name = "Eastward Water Velocity" ;
water_u:standard_name = "eastward_sea_water_velocity" ;
water_u:units = "m/s" ;
water_u:_FillValue = -30000s ;
water_u:missing_value = -30000s ;
water_u:scale_factor = 0.001f ;
water_u:add_offset = 0.f ;
water_u:NAVO_code = 17 ;
water_u:_Storage = "chunked" ;
water_u:_ChunkSizes = 1, 1, 976, 2196 ;
water_u:_DeflateLevel = 9 ;
water_u:_Endianness = "little" ;
short water_v(time, depth, lat, lon) ;
water_v:long_name = "Northward Water Velocity" ;
water_v:standard_name = "northward_sea_water_velocity" ;
water_v:units = "m/s" ;
water_v:_FillValue = -30000s ;
water_v:missing_value = -30000s ;
water_v:scale_factor = 0.001f ;
water_v:add_offset = 0.f ;
water_v:NAVO_code = 18 ;
water_v:_Storage = "chunked" ;
water_v:_ChunkSizes = 1, 1, 976, 2196 ;
water_v:_DeflateLevel = 9 ;
water_v:_Endianness = "little" ;
// global attributes:
:classification_level = "UNCLASSIFIED" ;
:distribution_statement = "Approved for public release.
Distribution unlimited." ;
:downgrade_date = "not applicable" ;
:classification_authority = "not applicable" ;
:institution = "Naval Oceanographic Office" ;
:source = "HYCOM archive file" ;
:history = "Fri Nov 20 10:01:58 2015: ncks -4 --deflate 9
-v water_u,water_v -d depth,1,2,1 hycom_glb_911_2015110200_t000_uv3z.nc
hycom_glb_911_2015110200_t000_uv3z-test.nc\narchv2ncdf3z" ;
:field_type = "instantaneous" ;
:Conventions = "CF-1.0 NAVO_netcdf_v1.0" ;
:NCO = "4.4.2" ;
:_Format = "netCDF-4" ;
data:
depth = 2, 4 ;
lat = -80, -79.9199981689453, -79.8399963378906, -79.7600021362305,
-79.6800003051758, -79.5999984741211, -79.5199966430664,
-79.4400024414062, -79.3600006103516, -79.2799987792969,
-79.1999969482422, -79.120002746582, -79.0400009155273,
-78.9599990844727, -78.879997253418, -78.8000030517578,
-78.7200012207031, -78.6399993896484, -78.5599975585938,
-78.4800033569336, -78.4000015258789, -78.3199996948242,
-78.2399978637695, -78.1600036621094, -78.0800018310547, -78,
-77.9199981689453, -77.8399963378906, -77.7600021362305,
-77.6800003051758, -77.5999984741211, -77.5199966430664,
-77.4400024414062, -77.3600006103516, -77.2799987792969,
-77.1999969482422, -77.120002746582, -77.0400009155273,
-76.9599990844727, -76.879997253418, -76.8000030517578,
-76.7200012207031, -76.6399993896484, -76.5599975585938,
-76.4800033569336, -76.4000015258789, -76.3199996948242,
-76.2399978637695, -76.1600036621094, -76.0800018310547, -76,
-75.9199981689453, -75.8399963378906, -75.7600021362305,
-75.6800003051758, -75.5999984741211, -75.5199966430664,
-75.4400024414062, -75.3600006103516, -75.2799987792969,
-75.1999969482422, -75.120002746582, -75.0400009155273,
-74.9599990844727, -74.879997253418, -74.8000030517578,
-74.7200012207031, -74.6399993896484, -74.5599975585938,
simonf@shaman [~/downl/tax]
16:52:45 $ ncdump -d -s /tmp/a.nc
ncdump: unreasonable value for float significant digits: 0
simonf@shaman [~/downl/tax]
16:52:50 $ ncdump -s /tmp/a.nc |more
netcdf a {
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
depth:_Storage = "chunked" ;
depth:_ChunkSizes = 2 ;
depth:_DeflateLevel = 9 ;
double lat(lat) ;
lat:long_name = "Latitude" ;
lat:standard_name = "latitude" ;
lat:units = "degrees_north" ;
lat:point_spacing = "even" ;
lat:axis = "Y" ;
lat:NAVO_code = 1 ;
lat:_Storage = "chunked" ;
lat:_ChunkSizes = 2001 ;
lat:_DeflateLevel = 9 ;
double lon(lon) ;
lon:long_name = "Longitude" ;
lon:standard_name = "longitude" ;
lon:units = "degrees_east" ;
lon:modulo = "360 degrees" ;
lon:axis = "X" ;
lon:NAVO_code = 2 ;
lon:_Storage = "chunked" ;
lon:_ChunkSizes = 4500 ;
lon:_DeflateLevel = 9 ;
double time(time) ;
time:long_name = "Valid Time" ;
time:units = "hours since 2000-01-01 00:00:00" ;
time:time_origin = "2000-01-01 00:00:00" ;
time:calendar = "gregorian" ;
time:axis = "T" ;
time:NAVO_code = 13 ;
time:_Storage = "chunked" ;
time:_ChunkSizes = 1 ;
time:_DeflateLevel = 9 ;
short water_u(time, depth, lat, lon) ;
water_u:long_name = "Eastward Water Velocity" ;
water_u:standard_name = "eastward_sea_water_velocity" ;
water_u:units = "m/s" ;
water_u:_FillValue = -30000s ;
water_u:missing_value = -30000s ;
water_u:scale_factor = 0.001f ;
water_u:add_offset = 0.f ;
water_u:NAVO_code = 17 ;
water_u:_Storage = "chunked" ;
water_u:_ChunkSizes = 1, 1, 976, 2196 ;
water_u:_DeflateLevel = 9 ;
water_u:_Endianness = "little" ;
short water_v(time, depth, lat, lon) ;
water_v:long_name = "Northward Water Velocity" ;
water_v:standard_name = "northward_sea_water_velocity" ;
water_v:units = "m/s" ;
water_v:_FillValue = -30000s ;
water_v:missing_value = -30000s ;
water_v:scale_factor = 0.001f ;
water_v:add_offset = 0.f ;
water_v:NAVO_code = 18 ;
water_v:_Storage = "chunked" ;
water_v:_ChunkSizes = 1, 1, 976, 2196 ;
water_v:_DeflateLevel = 9 ;
water_v:_Endianness = "little" ;
// global attributes:
:classification_level = "UNCLASSIFIED" ;
:distribution_statement = "Approved for public release.
Distribution unlimited." ;
:downgrade_date = "not applicable" ;
:classification_authority = "not applicable" ;
:institution = "Naval Oceanographic Office" ;
:source = "HYCOM archive file" ;
:history = "Fri Nov 20 10:01:58 2015: ncks -4 --deflate 9
-v water_u,water_v -d depth,1,2,1 hycom_glb_911_2015110200_t000_uv3z.nc
hycom_glb_911_2015110200_t000_uv3z-test.nc\narchv2ncdf3z" ;
:field_type = "instantaneous" ;
:Conventions = "CF-1.0 NAVO_netcdf_v1.0" ;
:NCO = "4.4.2" ;
:_Format = "netCDF-4" ;
data:
Second:
netcdf Oa20_radiance {
dimensions:
columns = 4865 ;
rows = 3682 ;
variables:
ushort Oa20_radiance(rows, columns) ;
Oa20_radiance:_FillValue = -1US ;
Oa20_radiance:add_offset = 0.f ;
Oa20_radiance:ancillary_variables = "Oa20_radiance_err" ;
Oa20_radiance:coordinates = "time_stamp altitude latitude
longitude" ;
Oa20_radiance:long_name = "TOA radiance for OLCI
acquisition band Oa20" ;
Oa20_radiance:scale_factor = 0.00326378f ;
Oa20_radiance:standard_name =
"toa_upwelling_spectral_radiance" ;
Oa20_radiance:units = "mW.m-2.sr-1.nm-1" ;
Oa20_radiance:valid_max = -2US ;
Oa20_radiance:valid_min = 0US ;
Oa20_radiance:_Storage = "chunked" ;
Oa20_radiance:_ChunkSizes = 1260, 1665 ;
Oa20_radiance:_DeflateLevel = 2 ;
Oa20_radiance:_Shuffle = "true" ;
Oa20_radiance:_Endianness = "little" ;
// global attributes:
:absolute_orbit_number = 3517U ;
:ac_subsampling_factor = 64US ;
:al_subsampling_factor = 1US ;
:comment = " " ;
:contact = "EOHElp@xxxxxxx" ;
:creation_time = "2016-10-20T13:00:25Z" ;
:institution = "SVL" ;
:netCDF_version = "4.2 of Jul 5 2012 17:07:43 $" ;
:product_name =
"S3A_OL_1_EFR____20161020T102624_20161020T102906_20161020T130025_0162_010_079_4319_SVL_O_NR_002.SEN3"
;
:references = "S3IPF PDS 004 - i2r1 - Product Data Format
Specification - OLCI, S3IPF PDS 002 - i1r6 - Product Data Format
Specification - Product Structures, S3IPF DPM 002 - i2r1 - Detailed
Processing Model - OLCI Level 1" ;
:resolution = "[ 270 294 ]" ;
:source = "IPF-OL-1-EO 06.04" ;
:start_time = "2016-10-20T10:26:24.238891Z" ;
:stop_time = "2016-10-20T10:29:06.207560Z" ;
:title = "OLCI Level 1b Product, Radiance Oa20 Data Set" ;
:_Format = "netCDF-4" ;
data:
Oa20_radiance =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, 13132, 13132, 13132, 13132,
14559, 14559, 14559, 14050, 14050, 14050, 12835, 12835, 12835, 11065,
11065, 10654, 10654, 10654, 11167, 11167, 11167, 14414, 14414, 14414,
15897, 15897, 15897, 13592, 13592, 13592, 11068, 11068, 11322, 11322,
11322, 11955, 11955, 11955, 11647, 11647, 11647, 11324, 11324, 11324,
8303, 8303, 7950, 7950, 7950, 8410, 8410, 8410, 7343, 7343, 7343,
11477,
11477, 12327, 12327, 12327, 12137, 12137, 12137, 13351, 13351, 13351,
14739, 14739, 14739, 14464, 14464, 14455, 14455, 14455, 16642, 16642,
16642, 16603, 16603, 16603, 17591, 17591, 17591, 18271, 18271, 18175,
18175, 18175, 16549, 16549, 16549, 13523, 13523, 13523, 11172, 11172,
10405, 10405, 10405, 10272, 10272, 10272, 10776, 10776, 10776, 10999,
10999, 13601, 13601, 13601, 17339, 17339, 17339, 17690, 17690, 17690,
14465, 14465, 11978, 11978, 11978, 12253, 12253, 12253, 13162, 13162,
13185, 13185, 13185, 11546, 11546, 11546, 11859, 11859, 11859, 11160,
11160, 10914, 10914, 10914, 13108, 13108, 13108, 13960, 13960, 13960,
16132, 16132, 18585, 18585, 18585, 19838, 19838, 19838, 17879, 17879,
16333, 16333, 16333, 15546, 15546, 15546, 13680, 13680, 13060, 13060,
13060, 15861, 15861, 15861, 18300, 18300, 18472, 18472, 18472, 17279,
17279, 17279, 15611, 15611, 14673, 14673, 14673, 11660, 11660, 11660,
11020, 11020, 9921, 9921, 9921, 8624, 8624, 8624, 8671, 8671, 10214,
10214, 10214, 10864, 10864, 10864, 11475, 11475, 13118, 13118, 13118,
13033, 13033, 13033, 11779, 11779, 13134, 13134, 13134, 14151, 14151,
14151, 13842, 13842, 12836, 12836, 12836, 13101, 13101, 16060, 16060,
16060, 16074, 16074, 16074, 13860, 13860, 13847, 13847, 13847, 16115,
16115, 17292, 17292, 17292, 20331, 20331, 20331, 21380, 21380, 19915,
19915, 19915, 17696, 17696, 16059, 16059, 16059, 16525, 16525, 16525,
netcdf Oa20_radiance {
dimensions:
columns = 4865 ;
rows = 3682 ;
variables:
ushort Oa20_radiance(rows, columns) ;
Oa20_radiance:_FillValue = -1US ;
Oa20_radiance:add_offset = 0.f ;
Oa20_radiance:ancillary_variables = "Oa20_radiance_err" ;
Oa20_radiance:coordinates = "time_stamp altitude latitude
long
itude" ;
Oa20_radiance:long_name = "TOA radiance for OLCI
acquisition b
and Oa20" ;
Oa20_radiance:scale_factor = 0.00326378f ;
Oa20_radiance:standard_name =
"toa_upwelling_spectral_radiance
" ;
Oa20_radiance:units = "mW.m-2.sr-1.nm-1" ;
Oa20_radiance:valid_max = -2US ;
Oa20_radiance:valid_min = 0US ;
Oa20_radiance:_Storage = "chunked" ;
Oa20_radiance:_ChunkSizes = 1260, 1665 ;
Oa20_radiance:_DeflateLevel = 2 ;
Oa20_radiance:_Shuffle = "true" ;
Oa20_radiance:_Endianness = "little" ;
// global attributes:
:absolute_orbit_number = 3517U ;
:ac_subsampling_factor = 64US ;
:al_subsampling_factor = 1US ;
:comment = " " ;
:contact = "EOHElp@xxxxxxx" ;
:creation_time = "2016-10-20T13:00:25Z" ;
:institution = "SVL" ;
:netCDF_version = "4.2 of Jul 5 2012 17:07:43 $" ;
:product_name =
"S3A_OL_1_EFR____20161020T102624_20161020T1029
06_20161020T130025_0162_010_079_4319_SVL_O_NR_002.SEN3" ;
:references = "S3IPF PDS 004 - i2r1 - Product Data Format
Spec
ification - OLCI, S3IPF PDS 002 - i1r6 - Product Data Format Specification
- P
roduct Structures, S3IPF DPM 002 - i2r1 - Detailed Processing Model - OLCI
Level 1" ;
:resolution = "[ 270 294 ]" ;
:source = "IPF-OL-1-EO 06.04" ;
:start_time = "2016-10-20T10:26:24.238891Z" ;
:stop_time = "2016-10-20T10:29:06.207560Z" ;
:title = "OLCI Level 1b Product, Radiance Oa20 Data Set" ;
:_Format = "netCDF-4" ;
On Tue, Dec 13, 2016 at 10:04 AM, Ed Hartnett <edwardjameshartnett@xxxxxxxxx
> wrote:
> Can you do an ncdump with the -s option, so we can see the chunksizes?
>
> On Tue, Dec 13, 2016 at 10:54 AM, Chris Barker <chris.barker@xxxxxxxx>
> wrote:
>
>> if I understand the question, this is read times -- is this in fact with
>> exactly the same files? In which case the chunking is already set.
>>
>> less performant decompression??
>>
>> sounds like the binary search is needed.
>>
>> Is EVERYTHING else the same? disk system, OS, etc?
>>
>> -CHB
>>
>>
>> On Tue, Dec 13, 2016 at 9:21 AM, Charlie Zender <zender@xxxxxxx> wrote:
>>
>>> Hello Simon,
>>>
>>> Since both files are netCDF4 compressed that
>>> means they use chunking. My wild guess is that
>>> different chunking defaults cause the observed
>>> change in dumping time. You can see the
>>> chunk sizes employed with ncdump -s or ncks --hdn,
>>> and you can play with the chunk sizes/policy
>>> with either.
>>>
>>> Charlie
>>> --
>>> Charlie Zender, Earth System Sci. & Computer Sci.
>>> University of California, Irvine 949-891-2429 )'(
>>>
>>>
>>> _______________________________________________
>>> NOTE: All exchanges posted to Unidata maintained email lists are
>>> recorded in the Unidata inquiry tracking system and made publicly
>>> available through the web. Users who post to any of the lists we
>>> maintain are reminded to remove any personal information that they
>>> do not want to be made public.
>>>
>>>
>>> netcdfgroup mailing list
>>> netcdfgroup@xxxxxxxxxxxxxxxx
>>> For list information or to unsubscribe, visit:
>>> http://www.unidata.ucar.edu/mailing_lists/
>>>
>>
>>
>>
>> --
>>
>> Christopher Barker, Ph.D.
>> Oceanographer
>>
>> Emergency Response Division
>> NOAA/NOS/OR&R (206) 526-6959 voice
>> 7600 Sand Point Way NE (206) 526-6329 fax
>> Seattle, WA 98115 (206) 526-6317 main reception
>>
>> Chris.Barker@xxxxxxxx
>>
>> _______________________________________________
>> NOTE: All exchanges posted to Unidata maintained email lists are
>> recorded in the Unidata inquiry tracking system and made publicly
>> available through the web. Users who post to any of the lists we
>> maintain are reminded to remove any personal information that they
>> do not want to be made public.
>>
>>
>> netcdfgroup mailing list
>> netcdfgroup@xxxxxxxxxxxxxxxx
>> For list information or to unsubscribe, visit:
>> http://www.unidata.ucar.edu/mailing_lists/
>>
>
>
> _______________________________________________
> NOTE: All exchanges posted to Unidata maintained email lists are
> recorded in the Unidata inquiry tracking system and made publicly
> available through the web. Users who post to any of the lists we
> maintain are reminded to remove any personal information that they
> do not want to be made public.
>
>
> netcdfgroup mailing list
> netcdfgroup@xxxxxxxxxxxxxxxx
> For list information or to unsubscribe, visit:
> http://www.unidata.ucar.edu/mailing_lists/
>
netcdfgroup archives: