NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.

To learn about what's going on, see About the Archive Site.

[netcdf-java] Variable shape problem & EOFException

I've found a problem in which cache/memory and disk shape information about variables will disagree with v2.2.22 of Java Netcdf library.
When you add a new value to a variable, automatically increasing the length of a dimension, subsequent reads can throw EOFException because RandomAccessFile is instructed to read more values than the file contains - the cached and actual shapes disagree.
I've created a runnable test case below to explain and demonstrate success and failure conditions.
I am getting around this now by not interleaving read/write operations on variables, but instead reading all variables' data to memory, then performing any writes I need to after. 

TestInsertRecord.java:


package com.metoceanengineers.datafeeds.netcdf.test;

import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;

import junit.framework.TestCase;
import ucar.ma2.Array;
import ucar.ma2.ArrayInt;
import ucar.ma2.DataType;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFileWriteable;

public class TestInsertRecord extends TestCase {
DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHMM"); protected NetcdfFileWriteable createNc(String prefix) throws IOException { File mainline = File.createTempFile(prefix+"-", ".nc"); NetcdfFileWriteable mainlineNc = NetcdfFileWriteable.createNew(mainline.getAbsolutePath(), false); 

       Dimension recordsDim = mainlineNc.addUnlimitedDimension("records");
Dimension timeDims[] = {recordsDim}; 
       Dimension var1Dims[] = {recordsDim}; // 1D
mainlineNc.addVariable("time", DataType.INT, timeDims); 
       mainlineNc.addVariable("var1", DataType.INT, var1Dims);

       mainlineNc.create();
       return mainlineNc;
   }


   protected String getNcInstance() throws Exception {

       NetcdfFileWriteable mainlineNc = createNc("testfile");
int[] origin = {0}; ArrayInt.D1 timeArr = new ArrayInt.D1(2); 
       timeArr.set(0, (int)dateFormat.parse("20071130 0924").getTime());
       timeArr.set(1, (int)dateFormat.parse("20071130 0926").getTime());
       mainlineNc.write("time", origin, timeArr);
ArrayInt.D1 var1Arr = new ArrayInt.D1(2); 
       var1Arr.set(0, 10);
       var1Arr.set(1, 12);
       mainlineNc.write("var1", origin, var1Arr);

       mainlineNc.close();

       return mainlineNc.getLocation();
   }

   /**
    * Append new data to end of existing variables.
    *
    * @throws Exception
    */
   public void testAppendWorksOk() throws Exception {
String ncFilename = getNcInstance(); NetcdfFileWriteable ncFile = NetcdfFileWriteable.openExisting(ncFilename, false); 

       /*
        * Append value (20071130 0924, 11) into (time, var1)
        */
ArrayInt.D1 newTimeValue = new ArrayInt.D1(1); newTimeValue.set(0, (int)dateFormat.parse("20071130 0925").getTime()); 

       ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
       newVarValue.set(0, 11);
int[] origin = {2}; 

       /* The first write will expand the variables,
        * but second write ok as we're just writing
        * and not reading */
ncFile.write("time", origin, newTimeValue); 
       ncFile.write("var1", origin, newVarValue);
assertEquals(3, ncFile.findDimension("records").getLength()); } 

   /**
    * Test insertion of a record in between the 2 existing
    * records by reading the existing tail, inserting new data
    * and re-appending.
    *
    * Triggers EOFException through interleaved read/writes
    *
    * @throws Exception
    */
   public void testInsertFails() throws Exception {
String ncFilename = getNcInstance(); NetcdfFileWriteable ncFile = NetcdfFileWriteable.openExisting(ncFilename, false); 

       ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
newTimeValue.set(0, (int)dateFormat.parse("20071130 0925").getTime()); 

       ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
       newVarValue.set(0, 11);
/* Going to insert at 1, so read existing value, 
        * write down new one, and re-append old tail.
        */
int[] insertPointOrigin = {1}; 
       int[] appendOrigin = {2};
       int[] shape = {1};
Array tailTime = ncFile.findVariable("time").read(insertPointOrigin, shape); 
       ncFile.write("time", insertPointOrigin, newTimeValue);
       ncFile.write("time", appendOrigin, tailTime);
/* Next line excepts - why? Because the last write above at 
        * records index 2 triggers an increase in the CACHED/MEMORY
        * length of all variables to 3, but on disk it's still the
        * original length 2.
        *
        *  Therefore we get EOFException.
        */
Array tailVar1 = ncFile.findVariable("var1").read(insertPointOrigin, shape); 
       ncFile.write("var1", insertPointOrigin, newVarValue);
       ncFile.write("var1", appendOrigin, tailVar1);
assertEquals(3, ncFile.findDimension("records").getLength()); } 

}
  • 2007 messages navigation, sorted by:
    1. Thread
    2. Subject
    3. Author
    4. Date
    5. ↑ Table Of Contents
  • Search the netcdf-java archives: