NOTICE: This version of the NSF Unidata web site (archive.unidata.ucar.edu) is no longer being updated.
Current content can be found at unidata.ucar.edu.

Re: [netcdf-java] Variable shape problem & EOFException

Hello. Is the lack of response here because this is not considered a bug? Is there a another recommended way perhaps to safely increase a dimension and invalidate the memory cache in the process? 

Nick


Nick Bower wrote:
I've found a problem in which cache/memory and disk shape information about variables will disagree with v2.2.22 of Java Netcdf library.
When you add a new value to a variable, automatically increasing the length of a dimension, subsequent reads can throw EOFException because RandomAccessFile is instructed to read more values than the file contains - the cached and actual shapes disagree.
I've created a runnable test case below to explain and demonstrate success and failure conditions.
I am getting around this now by not interleaving read/write operations on variables, but instead reading all variables' data to memory, then performing any writes I need to after. 

TestInsertRecord.java:


package com.metoceanengineers.datafeeds.netcdf.test;

import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;

import junit.framework.TestCase;
import ucar.ma2.Array;
import ucar.ma2.ArrayInt;
import ucar.ma2.DataType;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFileWriteable;

public class TestInsertRecord extends TestCase {
      DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHMM");
protected NetcdfFileWriteable createNc(String prefix) throws IOException { 
              File mainline = File.createTempFile(prefix+"-", ".nc");
NetcdfFileWriteable mainlineNc = NetcdfFileWriteable.createNew(mainline.getAbsolutePath(), false);
Dimension recordsDim = mainlineNc.addUnlimitedDimension("records"); 
              Dimension timeDims[] = {recordsDim};
       Dimension var1Dims[] = {recordsDim}; // 1D
              mainlineNc.addVariable("time", DataType.INT, timeDims);
       mainlineNc.addVariable("var1", DataType.INT, var1Dims);

       mainlineNc.create();
       return mainlineNc;
   }


   protected String getNcInstance() throws Exception {

       NetcdfFileWriteable mainlineNc = createNc("testfile");
              int[] origin = {0};
              ArrayInt.D1 timeArr = new ArrayInt.D1(2);
       timeArr.set(0, (int)dateFormat.parse("20071130 0924").getTime());
       timeArr.set(1, (int)dateFormat.parse("20071130 0926").getTime());
       mainlineNc.write("time", origin, timeArr);
              ArrayInt.D1 var1Arr = new ArrayInt.D1(2);
       var1Arr.set(0, 10);
       var1Arr.set(1, 12);
       mainlineNc.write("var1", origin, var1Arr);

       mainlineNc.close();

       return mainlineNc.getLocation();
   }
/** 
    * Append new data to end of existing variables.
    *
    * @throws Exception
    */
   public void testAppendWorksOk() throws Exception {
              String ncFilename = getNcInstance();
NetcdfFileWriteable ncFile = NetcdfFileWriteable.openExisting(ncFilename, false); 

       /*
        * Append value (20071130 0924, 11) into (time, var1)
        */
              ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
newTimeValue.set(0, (int)dateFormat.parse("20071130 0925").getTime()); 

       ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
       newVarValue.set(0, 11);
              int[] origin = {2};

       /* The first write will expand the variables,
        * but second write ok as we're just writing
        * and not reading */
              ncFile.write("time", origin, newTimeValue);
       ncFile.write("var1", origin, newVarValue);
assertEquals(3, ncFile.findDimension("records").getLength()); 
          }

   /**
    * Test insertion of a record in between the 2 existing
    * records by reading the existing tail, inserting new data
    * and re-appending.
    *
    * Triggers EOFException through interleaved read/writes
    *
    * @throws Exception
    */
   public void testInsertFails() throws Exception {
              String ncFilename = getNcInstance();
NetcdfFileWriteable ncFile = NetcdfFileWriteable.openExisting(ncFilename, false); 

       ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
newTimeValue.set(0, (int)dateFormat.parse("20071130 0925").getTime()); 

       ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
       newVarValue.set(0, 11);
              /* Going to insert at 1, so read existing value,
        * write down new one, and re-append old tail.
        */
              int[] insertPointOrigin = {1};
       int[] appendOrigin = {2};
       int[] shape = {1};
Array tailTime = ncFile.findVariable("time").read(insertPointOrigin, shape); 
       ncFile.write("time", insertPointOrigin, newTimeValue);
       ncFile.write("time", appendOrigin, tailTime);
/* Next line excepts - why? Because the last write above at 
        * records index 2 triggers an increase in the CACHED/MEMORY
        * length of all variables to 3, but on disk it's still the
        * original length 2.
        *
        *  Therefore we get EOFException.
        */
Array tailVar1 = ncFile.findVariable("var1").read(insertPointOrigin, shape); 
       ncFile.write("var1", insertPointOrigin, newVarValue);
       ncFile.write("var1", appendOrigin, tailVar1);
assertEquals(3, ncFile.findDimension("records").getLength()); 
          }

}