libodin-dev 1.8.8-2ubuntu1 / usr / include / odindata / data.h

/***************************************************************************
                          data.h  -  description
                             -------------------
    begin                : Fri Apr 6 2001
    copyright            : (C) 2000-2014 by Thies Jochimsen & Michael von Mengershausen
    email                : thies@jochimsen.de  mengers@cns.mpg.de
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#ifndef DATA_H
#define DATA_H

#ifndef TJUTILS_CONFIG_H
#define TJUTILS_CONFIG_H
#include <tjutils/config.h>
#endif

#include <tjutils/tjfeedback.h>

#ifdef ODIN_DEBUG
#define BZ_DEBUG
#endif




#include <blitz/blitz.h> // include 1st to overwrite mutex defines

// undef mutex defines
#ifdef BZ_MUTEX_DECLARE
#undef BZ_MUTEX_DECLARE
#endif
#ifdef BZ_MUTEX_INIT
#undef BZ_MUTEX_INIT
#endif
#ifdef BZ_MUTEX_LOCK
#undef BZ_MUTEX_LOCK
#endif
#ifdef BZ_MUTEX_UNLOCK
#undef BZ_MUTEX_UNLOCK
#endif
#ifdef BZ_MUTEX_DESTROY
#undef BZ_MUTEX_DESTROY
#endif

// do not use mutex in Blitz, even if pthread is available
// to get consistent behaviour across all platforms
#define BZ_MUTEX_DECLARE(name)
#define BZ_MUTEX_INIT(name)
#define BZ_MUTEX_LOCK(name)
#define BZ_MUTEX_UNLOCK(name)
#define BZ_MUTEX_DESTROY(name)



#include <blitz/array.h>
#ifdef HAVE_BLITZ_TINYVEC_ET
#include <blitz/tinyvec-et.h> // to use arithmetics of TinyVectors
#endif
using namespace blitz;


#include <tjutils/tjtypes.h>
#include <tjutils/tjprofiler.h>

#include <odinpara/protocol.h>

#include <odindata/converter.h>
#include <odindata/fileio_opts.h>


// some additional macros for complex numbers
BZ_DECLARE_FUNCTION_RET (float2real, STD_complex);
BZ_DECLARE_FUNCTION_RET (float2imag, STD_complex);
BZ_DECLARE_FUNCTION_RET (creal, float)
BZ_DECLARE_FUNCTION_RET (cimag, float)
BZ_DECLARE_FUNCTION_RET (cabs, float)
BZ_DECLARE_FUNCTION_RET (phase, float)
BZ_DECLARE_FUNCTION (logc)
BZ_DECLARE_FUNCTION (expc)
BZ_DECLARE_FUNCTION (conjc)

// Other useful macros
BZ_DECLARE_FUNCTION2(secureDivision)
BZ_DECLARE_FUNCTION (secureInv)


#ifdef STL_REPLACEMENT
BZ_DECLARE_ARRAY_ET_SCALAR_OPS(tjstd_complex)
#endif


/**
  * @addtogroup odindata
  * @{
  */

///////////////////////////////////////////////////////

struct FileMapHandle {
  FileMapHandle() : fd(-1), offset(0), refcount(1) {}
  int fd;
  LONGEST_INT offset; // in bytes
  int refcount;
  Mutex mutex; // protect refcount
};

///////////////////////////////////////////////////////

/**
  * This template class holds multidimensional data. In addition to functionality
  * provided by the Blitz Array class, it offers methods to read/write raw data
  * and and other file formats (nifti, vtk, ...) to/from a file.
  * In addition, uing a special constructor, the array can be associated with a block
  * of data on disk using the UNIX mmap functionality. In this case, any changes made
  * to the array will be automatically tranfered to the file. Please note that any
  * attempt to resize such an array will most likely cause a segmentation fault.
  *
  */
template <typename T, int N_rank>
class Data : public Array<T,N_rank> {

public:


/**
  * Constructs an array with the given dimensionality and initial value
  */
  Data(const TinyVector<int,N_rank>& dimvec, const T& val=0) : Array<T,N_rank>(dimvec), fmap(0) {(*this)=val;}


  // Variable extent constructors
  Data() : fmap(0) {}
  Data(int extent1) : Array<T,N_rank>(extent1), fmap(0)  {}
  Data(int extent1, int extent2) : Array<T,N_rank>(extent1,extent2), fmap(0)  {}
  Data(int extent1, int extent2,int extent3) : Array<T,N_rank>(extent1,extent2,extent3), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4) : Array<T,N_rank>(extent1,extent2,extent3,extent4), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4,int extent5) : Array<T,N_rank>(extent1,extent2,extent3,extent4,extent5), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4,int extent5,int extent6) : Array<T,N_rank>(extent1,extent2,extent3,extent4,extent5,extent6), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4,int extent5,int extent6,int extent7) : Array<T,N_rank>(extent1,extent2,extent3,extent4,extent5,extent6,extent7), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4,int extent5,int extent6,int extent7,int extent8) : Array<T,N_rank>(extent1,extent2,extent3,extent4,extent5,extent6,extent7,extent8), fmap(0)  {}
  Data(int extent1, int extent2,int extent3,int extent4,int extent5,int extent6,int extent7,int extent8,int extent9) : Array<T,N_rank>(extent1,extent2,extent3,extent4,extent5,extent6,extent7,extent8,extent9), fmap(0)  {}


/**
  * Maps the given file 'filename' into the newly created array via the the mmap function,
  * see the mmap man page for details. The mapping is abolish when the
  * array is destroyed (destructor). Other parameters:
  * - readonly : If true, file can only be read. If false, the file will be created if it does not exist
  * - shape    : The extent in each dimension
  * - offset   :  Skip first 'offset' bytes
  */
  Data(const STD_string& filename, bool readonly, const TinyVector<int,N_rank>& shape, LONGEST_INT offset=0 );

/**
  * Copy constructor
  */
  Data(const Data<T,N_rank>& d) : fmap(0) {Data<T,N_rank>::reference(d);}

/**
  * Copy constructor from Blitz::Array
  */
  Data(const Array<T,N_rank>& a) : Array<T,N_rank>(a), fmap(0) {}

/**
  * Constructor from an equivalent tjarray (\ref arrays). If rank of 'a' is smaller, extent will pe padded with 1 at front.
  */
  Data(const tjarray<tjvector<T>,T>& a) : fmap(0) { (*this)=a;}

/**
  * Constructor from expression template
  */
  template<class T_expr>
  Data(BZ_ETPARM(_bz_ArrayExpr<T_expr>) expr) : Array<T,N_rank>(expr), fmap(0) {}


/**
  * Destructor
  */
  ~Data();


/**
  * Assignment operator
  */
  Data<T,N_rank>& operator = (const Data<T,N_rank>& d) {Array<T,N_rank>::operator=(d); return *this;}

/**
  * Assignment operator from Blitz::Array
  */
  Data<T,N_rank>& operator = (const Array<T,N_rank>& a) {Array<T,N_rank>::operator=(a); return *this;}

/**
  * Fills all elements with 'val'
  */
  Data<T,N_rank>& operator = (const T& val) {Array<T,N_rank>::operator=(val); return *this;}

/**
  * Assignment operator for expression templates
  */
  template<class T_expr>
  inline Data<T,N_rank>& operator = (BZ_ETPARM(_bz_ArrayExpr<T_expr>) expr) {
    Data<T,N_rank>::reference(Data<T,N_rank>(expr)); // forwarding to constructor
    return *this;
  }

/**
  * Assignment from an equivalent tjarray (\ref arrays). If rank of 'a' is smaller, extent will pe padded with 1 at front.
  */
  Data<T,N_rank>& operator = (const tjarray<tjvector<T>,T>& a);


  // resolve ambiguities, appears to be obsolete with blitz-0.9 and later versions
//  Array<T,N_rank> operator + (const Data<T,N_rank>& b) const {return Array<T,N_rank>(Array<T,N_rank>(*this)+Array<T,N_rank>(b));}
//  Array<T,N_rank> operator - (const Data<T,N_rank>& b) const {return Array<T,N_rank>(Array<T,N_rank>(*this)-Array<T,N_rank>(b));}
//  Array<T,N_rank> operator * (const Data<T,N_rank>& b) const {return Array<T,N_rank>(Array<T,N_rank>(*this)*Array<T,N_rank>(b));}
//  Array<T,N_rank> operator / (const Data<T,N_rank>& b) const {return Array<T,N_rank>(Array<T,N_rank>(*this)/Array<T,N_rank>(b));}


  // resolve ambiguities, appears to be obsolete with blitz-0.9 and later versions
//  Array<T,N_rank> operator + (const T& v) const {return Array<T,N_rank>(Array<T,N_rank>(*this)+v);}
//  Array<T,N_rank> operator - (const T& v) const {return Array<T,N_rank>(Array<T,N_rank>(*this)-v);}
//  Array<T,N_rank> operator * (const T& v) const {return Array<T,N_rank>(Array<T,N_rank>(*this)*v);}
//  Array<T,N_rank> operator / (const T& v) const {return Array<T,N_rank>(Array<T,N_rank>(*this)/v);}



/**
  * Reads the array from a raw data file 'filename' of type 'format' which can be one of
  * 's8bit, u8bit, s16bit, u16bit, s32bit, u32bit, float or double'.
  * No resizing is performed prior to reading. Skip the first 'offset' bytes.
  * - Return value: 0 if succesful, otherwise -1
  */
  int read(const STD_string& format,const STD_string& filename, LONGEST_INT offset=0);


/**
  * Reads the array from a raw data file 'filename' of type 'T2'.
  * No resizing is performed prior to reading. Skip the first 'offset' bytes.
  * - Return value: 0 if succesful, otherwise -1
  */
  template<typename T2>
  int read(const STD_string& filename, LONGEST_INT offset=0);


/**
  * Writes the array to a raw data file 'filename' of type 'format' which can be one of
  * 's8bit, u8bit, s16bit, u16bit, s32bit, u32bit, float or double'.
  * If 'autoscale' is true, use full range of integer values (if applicable).
  * - Return value: 0 if succesful, otherwise -1
  */
  int write(const STD_string& format,const STD_string& filename, bool autoscale=true) const;


/**
  * Writes the array to a raw data file 'filename' of type 'T2'.
  * If 'autoscale' is true, use full range of integer values (if applicable).
  * - Return value: 0 if succesful, otherwise -1
  */
  template<typename T2>
  int write(const STD_string& filename, bool autoscale=true) const;



/**
  * Writes the array to a raw data file 'filename' of type the same type as the array is.
  *The mode specifies whether an already existing file is overwritten, or the data is appended.
  * - Return value: 0 if succesful, otherwise -1
  */
  int write(const STD_string& filename, fopenMode mode=overwriteMode) const;

/**
  * Writes the array as a formatted ASCII file  to
  * 'filename', one value per line. If 'pre' is
  * of the same size as this, its values are
  * inserted before the value on each line,
  * and the values of 'post' are inserted after
  * the values of this array on each line.
  * Return value is -1 if an error occurs.
  */
  int write_asc_file(const STD_string& filename, const Array<T,N_rank>& pre=defaultArray, const Array<T,N_rank>& post=defaultArray) const;

/**
  * reads in white-space separated data from ASCII file, no resizing is performed prior to reading
  * Return value is -1 if an error occurs.
  */
  int read_asc_file(const STD_string& filename);


/**
  * Writes data set to file 'filename' in the format automatically detected by the given file extension.
  * Extra options can be given by 'opts' and protocol settings can be overwritten by 'prot'.
  * Returns number of slices, or -1 if it fails.
  */
  int autowrite(const STD_string& filename, const FileWriteOpts& opts=FileWriteOpts(), const Protocol* prot=0) const;

/**
  * Reads data set from file 'filename' assuming the format automatically detected by the given file extension.
  * Extra options can be given by 'opts' and protocol settings will be stored in 'prot'.
  * The progress meter 'progmeter' can be optionally specified to monitor the progress of the operation.
  * Returns number of slices sucessfully read, or -1 if it fails.
  */
  int autoread(const STD_string& filename, const FileReadOpts& opts=FileReadOpts(), Protocol* prot=0, ProgressMeter* progmeter=0);

/**
  * Returns true if the array is associated with a file on disk
  */
  bool is_filemapped() const {return fmap;}

/**
  * Exports this array into an equivalent tjarray (\ref arrays)
  */
  operator tjarray<tjvector<T>,T> () const;


/**
  * Creates an index vector according to a linear index for the whole array
  * by taking the shape into account.
  */
  TinyVector<int, N_rank> create_index(unsigned long index) const;

/**
  * Creates a linear index from an index vector
  */
  unsigned long create_linear_index(const TinyVector<int, N_rank>& indexvec) const;

/**
  * Shift the data by 'shift' pixels in dimension 'shift_dim'. The
  * data is shifted cyclically, i.e. the values which are shifted
  * out at one edge of the array will be shifted in on the other edge.
  */
  void shift(unsigned int shift_dim, int shift);

/**
  * Interpolate (or extrapolate) the data in the array to the new shape.
  * The interpolation is performed dimension-by-dimension using the GSL spline interpolation
  * routines.
  * Add extra shift 'subpixel_shift' in units of pixels on the destination grid if 'subpixel_shift' is non-null.
  * if 'left_to_right' is set to true, interpolation starts at the most left
  * dimension, i.e. at those with the highest index, down to the lowest index.
  * If set to false, interpolation is performed vice versa.
  */
  void congrid(const TinyVector<int,N_rank>& newshape, const TinyVector<float,N_rank>* subpixel_shift=0, bool left_to_right=false);


/**
  * Converts the array to type 'T2', with dimensionality 'N_rank2' and stores the result in 'dst'.
  * 'dst' will be unique, i.e. no other references to the array exist.
  * In addition to changing 'dst', a reference to the same array is returned for convenience.
  * If 'autoscale' is true, use full range of integer values (if applicable).
  */
  template<typename T2, int N_rank2> Data<T2,N_rank2>& convert_to(Data<T2,N_rank2>& dst, bool autoscale=true) const;

  // specialization which uses reference if type/rank are same and scale/offset==1.0/0.0 to reduce memory usage
  Data<T,N_rank>& convert_to(Data<T,N_rank>& dst, bool autoscale=true) const;


/**
  * Makes array unique and continuous and returns pointer to data
  */
  T* c_array();


  // Reimplemented Blitz::Array functions
  void reference(const Data<T,N_rank>& d);


  // dummy objects used for default arguments
  static Array<T,N_rank> defaultArray;


 private:

  void interpolate1dim(unsigned int dim, int newsize, float subpixel_shift);

  void detach_fmap();

  FileMapHandle* fmap;

};

template <typename T, int N_rank>
Array<T,N_rank> Data<T,N_rank>::defaultArray;


/** @}
  */

///////////////////////////////////////////////////////////////////////////////

#ifdef STREAM_REPLACEMENT

template<typename T, int N_rank>
inline STD_ostream& operator << (STD_ostream& s,const Array<T,N_rank>&) {return s;}

template<typename T, int N_rank>
inline STD_ostream& operator << (STD_ostream& s,const TinyVector<T,N_rank>&) { return s;}

inline STD_ostream& operator << (STD_ostream& s,const Range&) { return s;}

#endif

///////////////////////////////////////////////////////////////////////////////


/*
 * Create n-dim index from linear 'index' for a given 'shape'
 */
template<int N_rank>
TinyVector<int,N_rank> index2extent(const TinyVector<int,N_rank>& shape, unsigned int index) {
  TinyVector<int,N_rank> result;
  unsigned int temp=index;
  for(int i=N_rank-1;i>=0;i--){
    result(i)=temp%shape(i);
    temp=temp/shape(i);
  }
  return result;
}


///////////////////////////////////////////////////////////////////////////////


// leave this function outside of class Data, otherwise it screws up
// when using optimization
template<typename T, int N_rank, typename T2>
void convert_from_ptr(Data<T,N_rank>& dst, const T2* src, const TinyVector<int, N_rank>& shape, bool autoscale=true) {
  Log<OdinData> odinlog("Data","convert_from_ptr");

  int dstsize=product(shape);
  int srcsize=dstsize*Converter::get_elements((T)0)/Converter::get_elements((T2)0);
  ODINLOG(odinlog,normalDebug) << "dstsize/srcsize=" << dstsize << "/" << srcsize << STD_endl;

  dst.resize(shape);

  Converter::convert_array(src, dst.c_array(), srcsize, dstsize, autoscale);
}

// specialization in case the type is the same
template<typename T, int N_rank>
void convert_from_ptr(Data<T,N_rank>& dst, const T* src, const TinyVector<int, N_rank>& shape) {
  dst.reference(Array<T,N_rank>((T*)src, shape, duplicateData));
}


////////////////////////////////////////////////////////////////////////////

template <typename T, int N_rank>
Data<T,N_rank>::Data(const STD_string& filename, bool readonly, const TinyVector<int,N_rank>& shape, LONGEST_INT offset ) : fmap(new FileMapHandle) {
  T* ptr=(T*)filemap(filename, (LONGEST_INT)product(shape)*sizeof(T), offset, readonly, fmap->fd);
  if(ptr && (fmap->fd)>=0) { // only negative file descriptor indicates filemap failure
    Array<T,N_rank>::reference(Array<T,N_rank>(ptr, shape, neverDeleteData));
    fmap->offset=offset;
  } else {
    delete fmap;
    fmap=0;
  }
}

////////////////////////////////////////////////////////////////////////////

template <typename T, int N_rank>
void Data<T,N_rank>::detach_fmap() {
  Log<OdinData> odinlog("Data","detach_fmap");
  if(fmap) {
    fmap->mutex.lock();
    (fmap->refcount)--;
    ODINLOG(odinlog,normalDebug) << "fd/offset/refcount=" << fmap->fd << "/" << fmap->offset << "/" << fmap->refcount << STD_endl;
    if(!(fmap->refcount)) {
      fileunmap(fmap->fd, Array<T,N_rank>::data(), (LONGEST_INT)Array<T,N_rank>::size()*sizeof(T), fmap->offset);
      fmap->mutex.unlock();
      delete fmap;
      fmap=0;
    }
    if(fmap) fmap->mutex.unlock();
  }
}

////////////////////////////////////////////////////////////////////////////

template <typename T, int N_rank>
Data<T,N_rank>::~Data() {
  detach_fmap();
}

////////////////////////////////////////////////////////////////////////////

template <typename T, int N_rank>
void Data<T,N_rank>::reference(const Data<T,N_rank>& d) {
  Log<OdinData> odinlog("Data","reference");
  detach_fmap();
  fmap=d.fmap;
  if(fmap) {
    MutexLock lock(fmap->mutex);
    (fmap->refcount)++;
    ODINLOG(odinlog,normalDebug) << "fmap->refcount=" << fmap->refcount << STD_endl;
  }
  Array<T,N_rank>::reference(d);
}

////////////////////////////////////////////////////////////////////////////

template <typename T, int N_rank>
Data<T,N_rank>& Data<T,N_rank>::operator = (const tjarray<tjvector<T>,T>& a) {
  Log<OdinData> odinlog("Data","=");
  if( a.dim() <= N_rank ) {
    ndim nn=a.get_extent();
    int npad=N_rank-nn.dim();
    ODINLOG(odinlog,normalDebug) << "npad=" << npad << STD_endl;
    for(int ipad=0; ipad<npad; ipad++) nn.add_dim(1, true); // pad at front
    ODINLOG(odinlog,normalDebug) << "nn(pad)" << nn << STD_endl;
    TinyVector<int,N_rank> tv;
    for(unsigned int i=0; i<N_rank; i++) tv(i)=nn[i];
    Data<T,N_rank>::resize(tv); // scope reuired by GCC 4.7
    for(unsigned int i=0;i<a.total();i++) (*this)(create_index(i))=a[i];
  } else ODINLOG(odinlog,errorLog) << "dimension mismatch: this=" << N_rank<< " < tjarray=" << a.dim() << STD_endl;
  return *this;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
int Data<T,N_rank>::read(const STD_string& format,const STD_string& filename, LONGEST_INT offset) {
  Log<OdinData> odinlog("Data","read");
  if(format==TypeTraits::type2label((u8bit)0))  return read<u8bit> (filename);
  if(format==TypeTraits::type2label((s8bit)0))  return read<s8bit> (filename);
  if(format==TypeTraits::type2label((u16bit)0)) return read<u16bit>(filename);
  if(format==TypeTraits::type2label((s16bit)0)) return read<s16bit>(filename);
  if(format==TypeTraits::type2label((u32bit)0)) return read<u32bit>(filename);
  if(format==TypeTraits::type2label((s32bit)0)) return read<s32bit>(filename);
  if(format==TypeTraits::type2label((float)0))  return read<float> (filename);
  if(format==TypeTraits::type2label((double)0)) return read<double>(filename);
  ODINLOG(odinlog,errorLog) << "Unable to read file " << filename << " with data type " << format << STD_endl;
  return -1;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
template<typename T2>
int Data<T,N_rank>::read(const STD_string& filename, LONGEST_INT offset) {
  Log<OdinData> odinlog("Data","read");
  LONGEST_INT fsize=filesize(filename.c_str())-offset;
  LONGEST_INT nelements_file=fsize/sizeof(T2);
  LONGEST_INT length=product(Array<T,N_rank>::shape());
  ODINLOG(odinlog,normalDebug) << "fsize / nelements_file / length = " << fsize << " / " << nelements_file << " / " << length << STD_endl;

  if(!length) return 0;

  if(nelements_file<length) {
    ODINLOG(odinlog,errorLog) << "Size of file " << filename << " to small for reading" << STD_endl;
    return -1;
  }

  STD_string srctype=TypeTraits::type2label((T2)0);
  STD_string dsttype=TypeTraits::type2label((T)0);
  ODINLOG(odinlog,normalDebug) << "srctype/dsttype=" << srctype << "/" << dsttype << STD_endl;

  TinyVector<int,N_rank> fileshape(Array<T,N_rank>::shape());
  fileshape(N_rank-1) *= (Converter::get_elements((T)0)/Converter::get_elements((T2)0)); // Adjust extent of last dim for copmlex data
  Data<T2,N_rank> filedata(filename, true, fileshape, offset);
  filedata.convert_to(*this);

  return 0;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
int Data<T,N_rank>::write(const STD_string& format, const STD_string& filename, bool autoscale) const {
  Log<OdinData> odinlog("Data","write");
  if(format==TypeTraits::type2label((u8bit)0))  return write<u8bit> (filename,autoscale);
  if(format==TypeTraits::type2label((s8bit)0))  return write<s8bit> (filename,autoscale);
  if(format==TypeTraits::type2label((u16bit)0)) return write<u16bit>(filename,autoscale);
  if(format==TypeTraits::type2label((s16bit)0)) return write<s16bit>(filename,autoscale);
  if(format==TypeTraits::type2label((u32bit)0)) return write<u32bit>(filename,autoscale);
  if(format==TypeTraits::type2label((s32bit)0)) return write<s32bit>(filename,autoscale);
  if(format==TypeTraits::type2label((float)0))  return write<float> (filename,autoscale);
  if(format==TypeTraits::type2label((double)0)) return write<double>(filename,autoscale);
  ODINLOG(odinlog,errorLog) << "Unable to write file " << filename << " with data type " << format << STD_endl;
  return -1;
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
int Data<T,N_rank>::write(const STD_string& filename, fopenMode mode) const {
  Log<OdinData> odinlog("Data","write");
  if(filename=="") return 0;

  FILE* file_ptr=FOPEN(filename.c_str(),modestring(mode));

  if(file_ptr==NULL) {
    ODINLOG(odinlog,errorLog) << "unable to create/open file >" << filename << "< - " << lasterr() << STD_endl;
    return -1;
  }

  Data<T,N_rank> data_copy(*this); // for contig memory

  LONGEST_INT nmemb=Array<T,N_rank>::numElements();
  LONGEST_INT count=fwrite(data_copy.c_array(),sizeof(T),nmemb,file_ptr);
  if(count!=nmemb) {
    ODINLOG(odinlog,errorLog) << "unable to fwrite to file >" << filename << "< - " << lasterr() << STD_endl;
    return -1;
  }

  if(file_ptr!=NULL) fclose(file_ptr);
  return 0;
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
template<typename T2>
  int Data<T,N_rank>::write(const STD_string& filename, bool autoscale) const {
  Log<OdinData> odinlog("Data","write");

  rmfile(filename.c_str()); // remove old file to get new file size with mmap

  ODINLOG(odinlog,normalDebug) << "mem(pre): " << Profiler::get_memory_usage() << STD_endl;
  Data<T2,N_rank> converted_data; convert_to(converted_data,autoscale);
  ODINLOG(odinlog,normalDebug) << "mem(cnv): " << Profiler::get_memory_usage() << STD_endl;
  Data<T2,N_rank> filedata(filename,false,converted_data.shape());
  ODINLOG(odinlog,normalDebug) << "mem(map): " << Profiler::get_memory_usage() << STD_endl;
  filedata=converted_data;

  return 0;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
int Data<T,N_rank>::write_asc_file(const STD_string& filename, const Array<T,N_rank>& pre, const Array<T,N_rank>& post) const {
  bool have_pre=false;
  bool have_post=false;

  Data<T,N_rank> pre_data(pre);
  Data<T,N_rank> post_data(post);
  int n=Array<T,N_rank>::numElements();

  if(pre_data.numElements()==n) have_pre=true;
  if(post_data.numElements()==n) have_post=true;

  STD_ofstream ofs(filename.c_str());
  if(ofs.bad()) return -1;

  T val;

  for(int i=0; i<n; i++) {
    if(have_pre) {
      val=pre_data(pre_data.create_index(i));
      ofs << val << " ";
    }
    val=(*this)(create_index(i));
    ofs << val;
    if(have_post) {
      val=post_data(post_data.create_index(i));
      ofs << " " << val;
    }
    ofs << "\n";
  }

  ofs.close();
  return 0;
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
int Data<T,N_rank>::read_asc_file(const STD_string& filename) {
  STD_ifstream ifs(filename.c_str());
  if(ifs.bad()) return -1;

  STD_string valstr;
  for(int i=0; i<Array<T,N_rank>::numElements(); i++) {
    if(ifs.bad()) return -1;
    ifs >> valstr;
    TypeTraits::string2type(valstr,(*this)(create_index(i)));
  }

  ifs.close();
  return 0;
}


////////////////////////////////////////////////////////////////////////////

// Interface to FileIO functionality
int fileio_autowrite(const Data<float,4>& data, const STD_string& filename, const FileWriteOpts& opts, const Protocol* prot);
int fileio_autoread(Data<float,4>& data, const STD_string& filename, const FileReadOpts& opts, Protocol* prot, ProgressMeter* progmeter);


template<typename T,int N_rank>
int Data<T,N_rank>::autowrite(const STD_string& filename, const FileWriteOpts& opts, const Protocol* prot) const {
  Data<float,4> filedata; convert_to(filedata);
  return fileio_autowrite(filedata, filename, opts, prot);
}

template<typename T,int N_rank>
int Data<T,N_rank>::autoread(const STD_string& filename, const FileReadOpts& opts, Protocol* prot, ProgressMeter* progmeter) {
  Data<float,4> filedata;
  int result=fileio_autoread(filedata, filename, opts, prot, progmeter);
  if(result>0) filedata.convert_to(*this);
  return result;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
Data<T,N_rank>::operator tjarray<tjvector<T>,T> () const {
   tjarray<tjvector<T>,T>  a;
   ndim nn(N_rank);
   for(unsigned int i=0; i<N_rank; i++) nn[i]=Array<T,N_rank>::extent(i);
   a.redim(nn);
   for(unsigned int i=0;i<a.total();i++) a[i]=(*this)(create_index(i));
   return a;
}



////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
template<typename T2, int N_rank2>
Data<T2,N_rank2>& Data<T,N_rank>::convert_to(Data<T2,N_rank2>& dst, bool autoscale) const {
  Log<OdinData> odinlog("Data","convert_to");

  TinyVector<int,N_rank2> newshape; newshape=1;
  for(int i=0; i<(N_rank-N_rank2+1); i++) { // collapse extra dims into first dim of new shape, including first dim
   int srcindex=i;
   if(srcindex>=0 && srcindex<N_rank) newshape(0)*=Array<T,N_rank>::extent(srcindex);
  }
  for(int i=0; i<(N_rank2-1); i++) { // Fill new shape with extents of original dim, except for the first dim
    int srcindex=N_rank-N_rank2+1+i;
    if(srcindex>=0 && srcindex<N_rank) newshape(1+i)=Array<T,N_rank>::extent(srcindex);
  }

  // modify last dimension to account for different element sizes
  newshape(N_rank2-1)=newshape(N_rank2-1)*Converter::get_elements((T)0)/Converter::get_elements((T2)0);

  dst.resize(newshape); // dst will be unique now

  Data<T,N_rank> src_copy(*this); // make read/write copy of this

  Converter::convert_array(src_copy.c_array(), dst.c_array(), src_copy.numElements(), dst.numElements(), autoscale);

  return dst;
}


// specialization which uses reference if type/rank are same and no scaling is necessary to reduce memory usage
template<typename T,int N_rank>
Data<T,N_rank>& Data<T,N_rank>::convert_to(Data<T,N_rank>& dst, bool autoscale) const {
  Log<OdinData> odinlog("Data","convert_to");
  if(!autoscale || !std::numeric_limits<T>::is_integer) {
    ODINLOG(odinlog,normalDebug) << "Using reference" << STD_endl;
    dst.reference(*this);
  } else {
    Data<T,N_rank> src_copy(*this); // make read/write copy of this
    dst.resize(src_copy.shape()); // dst will be unique now
    Converter::convert_array(src_copy.c_array(), dst.c_array(), src_copy.numElements(), dst.numElements(), autoscale);
  }
  return dst;
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
TinyVector<int, N_rank> Data<T,N_rank>::create_index(unsigned long index) const {
  return index2extent<N_rank>(Array<T,N_rank>::shape(), index);
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
unsigned long Data<T,N_rank>::create_linear_index(const TinyVector<int, N_rank>& indexvec) const {
  unsigned long totalIndex=0,subsize;
  TinyVector<int, N_rank> nn(Array<T,N_rank>::extent());
  for(unsigned long i=0;i<N_rank;i++) {
    nn(i)=1;
    subsize=product(nn);
    if(!subsize) subsize=1;
    totalIndex+=subsize*indexvec(i);
  }
  return totalIndex;
}

////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
void Data<T,N_rank>::shift(unsigned int shift_dim, int shift) {
  Log<OdinData> odinlog("Data","shift");

  if(!shift) return;

  if( shift_dim >= N_rank ){
    ODINLOG(odinlog,errorLog) << "shift dimension(" << shift_dim << ") >= rank of data (" << N_rank << ") !\n";
    return;
  }

  int shift_extent=Array<T,N_rank>::extent(shift_dim);
  int abs_shift=abs(shift);
  if(shift_extent < abs_shift){
    ODINLOG(odinlog,errorLog) << "extent(" << shift_extent << ") less than shift(" << abs_shift << ") !\n";
    return;
  }

  Data<T,N_rank> data_copy(Array<T,N_rank>::copy());

  TinyVector<int,N_rank> index;
  for(int i=0; i<Array<T,N_rank>::numElements(); i++) {
    index=create_index(i);
    T val=data_copy(index);
    int shiftindex=index(shift_dim)+shift;
    if(shiftindex>=shift_extent) shiftindex-=shift_extent;
    if(shiftindex<0)             shiftindex+=shift_extent;
    index(shift_dim)=shiftindex;
    (*this)(index)=val;
  }

}



////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
void Data<T,N_rank>::congrid(const TinyVector<int,N_rank>& newshape, const TinyVector<float,N_rank>* subpixel_shift, bool left_to_right) {
  Log<OdinData> odinlog("Data","congrid");
  for(int i=0;i<N_rank;i++) {
    ODINLOG(odinlog,normalDebug) << "oldshape/newshape(" << i << ")=" << Array<T,N_rank>::shape()(i) << "/" << newshape(i) << STD_endl;
    if(subpixel_shift) {
      ODINLOG(odinlog,normalDebug) << "subpixel_shift(" << i << ")=" << (*subpixel_shift)(i) << STD_endl;
    }
  }

  for(int irank=0; irank<N_rank; irank++) {
    int dim=irank;
    if(!left_to_right) dim=N_rank-1-irank;
    float shift=0.0;
    if(subpixel_shift) shift=(*subpixel_shift)(dim);
    interpolate1dim(dim,newshape(dim),shift);
  }
}

////////////////////////////////////////////////////////////////////////////


template<typename T,int N_rank>
void Data<T,N_rank>::interpolate1dim(unsigned int dim, int newsize, float subpixel_shift) {
  Log<OdinData> odinlog("Data","interpolate1dim");
  ODINLOG(odinlog,normalDebug) << "dim/newsize=" << dim << "/" << newsize << STD_endl;

  if(newsize==Array<T,N_rank>::shape()(dim) && subpixel_shift==0.0) {
    ODINLOG(odinlog,normalDebug) << "interpolation not required" << STD_endl;
    return;
  }

  if(dim>=N_rank) {
    ODINLOG(odinlog,errorLog) << "dim is larger than N_rank" << STD_endl;
    return;
  }
  if(newsize<0) {
    ODINLOG(odinlog,errorLog) << "newsize is negative" << STD_endl;
    return;
  }

  Array<T,N_rank> olddata(*this);
  olddata.makeUnique();

  TinyVector<int,N_rank> oldshape(olddata.shape());
  int oldsize=oldshape(dim);

  TinyVector<int,N_rank> newshape(oldshape);
  newshape(dim)=newsize;
  Data<T,N_rank>::resize(newshape); // scope reuired by GCC 4.7

  // This holds the the shape of the subspace which is orthogonal to direction 'dim'
  TinyVector<int,N_rank> ortho_shape(oldshape);
  ortho_shape(dim)=1;

  // The total number of elements in the orthogonal subspace
  unsigned long n_ortho=product(ortho_shape);

  ODINLOG(odinlog,normalDebug) << "n_ortho=" << n_ortho << STD_endl;

  TinyVector<int,N_rank> indexvec;
  T* oldoneline=new T[oldsize];

  for(unsigned long iortho=0; iortho<n_ortho; iortho++) {

    indexvec=index2extent<N_rank>(ortho_shape,iortho);

    for(int j=0; j<oldsize; j++) {
      indexvec(dim)=j;
      oldoneline[j]=olddata(indexvec);
    }

    T* newoneline=interpolate1D(oldoneline,oldsize,newsize,subpixel_shift);

    for(int j=0; j<newsize; j++) {
      indexvec(dim)=j;
      (*this)(indexvec)=newoneline[j];
    }

    delete[] newoneline;
  }

  delete[] oldoneline;
}


////////////////////////////////////////////////////////////////////////////

template<typename T,int N_rank>
T* Data<T,N_rank>::c_array() {
  Log<OdinData> odinlog("Data","c_array");

  bool need_copying=false;

  // check storage order
  TinyVector<int,N_rank>ord=Array<T,N_rank>::ordering();
  for(int i=0; i<N_rank-1; i++) {
    if(ord(i)<ord(i+1)) need_copying=true;
  }

  // check storage direction
  for(int i=0; i<N_rank; i++) {
    if(!Array<T,N_rank>::isRankStoredAscending(i)) need_copying=true;
  }

  // check for slicing
  if(!Array<T,N_rank>::isStorageContiguous()) need_copying=true;

  if(need_copying) {
    ODINLOG(odinlog,normalDebug) << "need_copying" << STD_endl;
    Data<T,N_rank> tmp(Array<T,N_rank>::shape());
    tmp=(*this); // reference(copy()) would not be enough since it would preserve ordering
    reference(tmp);
  }

  return Array<T,N_rank>::data();
}

#endif