NetCDF table

Storing tables in NetCDF4

Writing a table using Python

The following example shows how to store a table of 10 records with 8 members :

name	ADCcount	grid_i	grid_j	pressure	energy	idnumber	pressure2
16-character String	Unsigned short integer	32-bit integer	32-bit integer	float (single-precision)	double (double-precision)	Signed 64-bit integer	2-dim table of float (2*3)

The script has been run on Niagara with the following modules :

module load anaconda3/5.2.0

from netCDF4 import Dataset
from netCDF4 import chartostring, stringtoarr
import numpy

f = Dataset('particles.nc','w',format='NETCDF4')

size = 10

Particle = numpy.dtype([('name', 'S1', 16),                       # 16-character String                                                
                        ('ADCcount',numpy.uint16),                # Unsigned short integer                                             
                        ('grid_i',numpy.int32),                   # 32-bit integer                                                     
                        ('grid_j',numpy.int32),                   # 32-bit integer                                                     
                        ('pressure',numpy.float32),               # float  (single-precision)                                          
                        ('energy',numpy.float64),                 # double (double-precision)                                          
                        ('idnumber',numpy.int64),                 # Signed 64-bit integer                                              
                        ('pressure2' , numpy.float32 , (2,3) )    # array of floats (single-precision) table 2 lines * 3 columns       
                        ])

Particle_t = f.createCompoundType(Particle,'Particle')

f.createDimension('NRecords',None)
v = f.createVariable('Data',Particle_t,'NRecords')
data = numpy.empty(size,Particle)

for i in range(10):
    data['name'][i] = '{:10s}{:6d}'.format('Particle:', i))
    data['ADCcount'][i] = (i * 256) % (1 << 16)
    data['grid_i'][i] = i
    data['grid_j'][i] = 10 - i
    data['pressure'][i] = float(i*i)
    data['energy'][i] = float(data['pressure'][i] ** 4)
    data['idnumber'][i] = i * (2 ** 34)
    data['pressure2'][i] = [
        [0.5+float(i),1.5+float(i),2.5+float(i)],
        [-1.5+float(i),-2.5+float(i),-3.5+float(i)]]
#Fill data in File                                                                                                                     
v[:] = data

f.close()

The NetCDF file can be dumped using : ncdump particles.nc. The ncdump command is available through the netcdf module :

$ module load intel/2018.2
$ module load hdf5/1.8.20
$ module load netcdf/4.6.1
$
$ ncdump particles.nc
netcdf particles {
types:
  compound Particle {
    char name(16) ;
    ushort ADCcount ;
    int grid_i ;
    int grid_j ;
    float pressure ;
    double energy ;
    int64 idnumber ;
    float pressure2(2, 3) ;
  }; // Particle
dimensions:
	NRecords = UNLIMITED ; // (10 currently)
variables:
	Particle Data(NRecords) ;
data:

 Data = 
    {{"Particle:      0"}, 0, 0, 10, 0, 0, 0, {0.5, 1.5, 2.5, -1.5, -2.5, -3.5}}, 
    {{"Particle:      1"}, 256, 1, 9, 1, 1, 17179869184, {1.5, 2.5, 3.5, -0.5, -1.5, -2.5}}, 
    {{"Particle:      2"}, 512, 2, 8, 4, 256, 34359738368, {2.5, 3.5, 4.5, 0.5, -0.5, -1.5}}, 
    {{"Particle:      3"}, 768, 3, 7, 9, 6561, 51539607552, {3.5, 4.5, 5.5, 1.5, 0.5, -0.5}}, 
    {{"Particle:      4"}, 1024, 4, 6, 16, 65536, 68719476736, {4.5, 5.5, 6.5, 2.5, 1.5, 0.5}}, 
    {{"Particle:      5"}, 1280, 5, 5, 25, 390625, 85899345920, {5.5, 6.5, 7.5, 3.5, 2.5, 1.5}}, 
    {{"Particle:      6"}, 1536, 6, 4, 36, 1679616, 103079215104, {6.5, 7.5, 8.5, 4.5, 3.5, 2.5}}, 
    {{"Particle:      7"}, 1792, 7, 3, 49, 5764801, 120259084288, {7.5, 8.5, 9.5, 5.5, 4.5, 3.5}}, 
    {{"Particle:      8"}, 2048, 8, 2, 64, 16777216, 137438953472, {8.5, 9.5, 10.5, 6.5, 5.5, 4.5}}, 
    {{"Particle:      9"}, 2304, 9, 1, 81, 43046721, 154618822656, {9.5, 10.5, 11.5, 7.5, 6.5, 5.5}} ;
}

Reading the table with a C++ code

The code has been compiled and tested on Niagara with the following modules :

module load intel/2018.2 hdf5/1.8.20 netcdf/4.6.1
icc -I$SCINET_NETCDF_ROOT/include Test.cpp -o Test -lnetcdf_c++4 -lnetcdf

Test.cpp :

#include <iostream>
#include <netcdf>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

using namespace std;

#define FILE_NAME "particles.nc"
#define DIM_LEN 10 //number of records in file 
int main(void)
{
  typedef struct Particle
  {
    char   name[16];
    unsigned short int    ADCcount;
    int grid_i;
    int grid_j;
    float pressure;
    double energy ;
    long  idnumber;
    float pressure2[2][3];
  } Particle;

  //Definition of the data_in variable
  Particle data_in[DIM_LEN];

  //Initialization of the variable
  for (int i=0; i<DIM_LEN; i++){
    data_in[i].ADCcount=0;
    data_in[i].grid_i=0;
    data_in[i].grid_j=0;
    data_in[i].pressure=0.;
    data_in[i].energy=0.;
    for (int j=0; j<2; j++){
      for (int k=0; k<3; k++){
        data_in[i].pressure2[j][k]=0.;
      }
    }
  }

  //Variables needed to open the NetCDF file :
  int ncid, typeidd, varid;
  int dimid;
  int dimids[] = {0}, fieldid;

  nc_def_dim(ncid, "NRecords", DIM_LEN, &dimid);
  nc_def_var(ncid, "Data", typeidd, 1, dimids, &varid);

  //Open the NetCDF file :
  if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) cout<<"ERROR"<<endl; //NC_NOWRITE for read only, NC_WRITE for read and write
  //Read the data and fill the variable data_in :
  if (nc_get_var(ncid, varid, data_in)) cout<<"ERROR"<<endl;

  for (int i=0; i<DIM_LEN; i++){
    std::cout<<"ADCcount = "<<data_in[i].ADCcount
             <<" ,idnumber = "<<data_in[i].idnumber
             <<" ,grid_i = "<<data_in[i].grid_i
             <<" ,grid_j = "<<data_in[i].grid_j
             <<" ,pressure = "<<data_in[i].pressure
             <<" ,energy = "<<data_in[i].energy
             <<" ,name = "<<data_in[i].name
             <<std::endl;
    for(int j=0;j<2;j++){
      std::cout<<data_in[i].pressure2[j][0]<<" "<<data_in[i].pressure2[j][1]<<" "<<data_in[i].pressure2[j][2]<<std::endl;
    }
  }
cout << "*** SUCCESS reading example file "<<FILE_NAME<<"!" << endl;
  return 0;
}

Using Parallel netCDF

The compound format (actually HDF5) isn't compatible with parallel-netcdf. There are parallel I/O facilities in NetCDF-4. To use the parallel features of netcdf-4, use the call nc_open_par().