NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.
To learn about what's going on, see About the Archive Site.
NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.
To learn about what's going on, see About the Archive Site.
Hi Ed, Ed Hartnett wrote:
Mario Emmenlauer <mario@xxxxxxxxxxxxx> writes:Hi, Roy Mendelssohn wrote:See http://www.unidata.ucar.edu/software/netcdf/docs/faq.html#Large%20File%20Support0. It has been possible since netcdf3.6with severe bugs, as posted by me to this mailing list on 09/12/2007 (i.e. [netcdfgroup] possible bug in 3.6.2 with variables > 4GB). For variable types NC_BYTE, NC_CHAR and NC_SHORT it crashes badly. I have actually never received an answer to my bug report, don't know why nobody cared :-/We care very much! But apparently your email passed us by without action, which is not good. Sorry about that. :-(
I did not mean to sound so harsh, I was just very much concerned back then that nobody ever recognized such a severe bug. I mean, writing files > 4GB did never actually work for datatypes < 32bit, did it?
For your bug, what are you trying to do, write the extra large last variable?
We have a library that interfaces blitz++ with netcdf, and that
provides simple save_array and load_array functions. The save_array
call for 3D datasets has (without the debug output messages) the
following form:
template< class DataType, int Rank>
void
BlitzNCFile::save_array( const blitz::Array<DataType,Rank> &arr,
const char *varName)
{
/*---------------------------------------------------------------------
* Check if variable already exists and has correct dimensions
*---------------------------------------------------------------------*/
NcError errorHandling( NcError::silent_nonfatal);
NcVar *outVar = get_var( varName);
if( outVar != 0)
{
if( outVar->num_dims() != 3
|| outVar->type() != get_NcType( DataType() )
|| outVar->get_dim(0)->size() != arr.extent(0)
|| outVar->get_dim(1)->size() != arr.extent(1)
|| outVar->get_dim(2)->size() != (arr.extent(2)
* numberComponents<DataType>()))
{
BlitzNCSaveError err;
err << "Can't save data: variable `" << varName
<< "' already exists with different dimensions or type\n"
<< "voxelblock has type " << get_NcTypeName( DataType() )
<< " and dims (" << arr.extent(0) << "," << arr.extent(1)
<< "," << arr.extent(2) << ")\n";
throw( err);
}
}
else
{
/*-------------------------------------------------------------------
* Create new variable (3D)
* check if dimensions exist or create new
*-------------------------------------------------------------------*/
std::vector<std::string> defaultDimNames;
const std::vector<std::string> *dimNames
char tmp[256];
sprintf( tmp, "level_%d", arr.extent(0));
defaultDimNames.push_back( tmp);
sprintf( tmp, "row_%d", arr.extent(1));
defaultDimNames.push_back( tmp);
sprintf( tmp, "col_%d", arr.extent(2));
defaultDimNames.push_back( std::string(tmp)
+ colDimNameExtension_nc<DataType>());
dimNames = &defaultDimNames;
// add the three dimensions, all with error checking
NcDim* levelDim = get_dim( (*dimNames)[0].c_str());
if( levelDim == 0)
{
levelDim = add_dim( (*dimNames)[0].c_str(), arr.extent(0));
if( levelDim == 0)
{
BlitzNCSaveError err;
err << fileName() << ": " << nc_strerror( errorHandling.get_err());
throw( err);
}
}
else
{
if( levelDim->size() != arr.extent(0))
{
BlitzNCSaveError err;
err << "Can't save data: Variable `" << varName
<< "' has " << levelDim->size()
<< " levels and voxelblock has "
<< arr.extent(0) << " levels!\n";
throw( err);
}
}
NcDim* rowDim = get_dim( (*dimNames)[1].c_str());
if( rowDim == 0)
{
rowDim = add_dim( (*dimNames)[1].c_str(), arr.extent(1));
if( rowDim == 0)
{
BlitzNCSaveError err;
err << fileName() << ": " << nc_strerror( errorHandling.get_err());
throw( err);
}
}
else
{
if( rowDim->size() != arr.extent(1))
{
BlitzNCSaveError err;
err << "Can't save data: Variable `" << varName
<< "' has " << rowDim->size() << " rows and voxelblock has "
<< arr.extent(1) << " rows!\n";
throw( err);
}
}
NcDim* colDim = get_dim( (*dimNames)[2].c_str());
if( colDim == 0)
{
colDim = add_dim( (*dimNames)[2].c_str(),
arr.extent(2) * numberComponents<DataType>());
if( colDim == 0)
{
BlitzNCSaveError err;
err << fileName() << ": " << nc_strerror( errorHandling.get_err());
throw( err);
}
}
else
{
if( colDim->size() != arr.extent(2) * numberComponents<DataType>())
{
BlitzNCSaveError err;
err << "Can't save data: Variable `" << varName
<< "' has " << colDim->size() << " cols and voxelblock has "
<< arr.extent(2) * numberComponents<DataType>()
<< " cols!\n";
throw( err);
}
}
// finally, we can add the varibale
outVar = add_var( varName, get_NcType( DataType()),
levelDim, rowDim, colDim);
if( outVar == 0)
{
BlitzNCSaveError err;
err << fileName() << ": " << nc_strerror( errorHandling.get_err());
throw( err);
}
}
/*---------------------------------------------------------------------
* Write data to variable
*---------------------------------------------------------------------*/
// Note: we use this on x86_64 linux hosts, where long is 64 bit
// (however correctly this should all be ssize_t)
int nlevels = arr.extent(0);
long nrows = arr.extent(1);
long ncols = arr.extent(2) * numberComponents<DataType>();
const DataType *data = arr.data();
long picsize = nrows*arr.extent(2);
for( int i = 0; i < nlevels; i++)
{
outVar->set_cur(i,0,0);
outVar->put( ncCast( data), 1, nrows, ncols);
data += picsize;
}
}
I do not know which part actually triggers the assertion, and I'm out
of office until next week. Do you want me to test with valgrind when
I'm back?
Cheers,
Mario
netcdfgroup archives: