/***************************************************************************
                          datavector.cpp  -  a vector which gets its data from
                          a datasource.
                             -------------------
    begin                : Fri Sep 22 2000
    copyright            : (C) 2000-2010 by C. Barth Netterfield
    email                : netterfield@astro.utoronto.ca
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *   Permission is granted to link with any opensource library             *
 *                                                                         *
 ***************************************************************************/
#include "datavector.h"

#include <assert.h>
#include <math.h>
#include <stdlib.h>

#include <QDebug>
#include <QXmlStreamWriter>

#include "datacollection.h"
#include "debug.h"
#include "datasource.h"
#include "math_kst.h"
#include "objectstore.h"
#include "updatemanager.h"
#include "vectorscriptinterface.h"

// ReqNF <=0 means read from ReqF0 to end of File
// ReqF0 < means start at EndOfFile-ReqNF.
//
// ReqNF      ReqF0      Action
//  < 1        >=0       read from ReqF0 to end of file
//  < 1        < 0       illegal: fixed in checkIntegrity
//    1         ??       illegal: fixed in checkIntegrity
//  > 1        < 0       read the last ReqNF frames from the file
//  > 1        >=0       Read ReqNF frames starting at frame ReqF0

namespace Kst {

const QString DataVector::staticTypeString = "Data Vector";
const QString DataVector::staticTypeTag = "datavector";

const int INVALIDS_PER_RESET = 5;

DataVector::DataInfo::DataInfo() :
    frameCount(-1),
    samplesPerFrame(-1)
{
}


DataVector::DataInfo::DataInfo(int fc, int spf) :
    frameCount(fc),
    samplesPerFrame(spf)
{
}

/** Create a DataVector: raw data from a file */
DataVector::DataVector(ObjectStore *store)
: Vector(store), DataPrimitive(this) {

  _saveable = true;
  //_dontUseSkipAccel = false;
  _numSamples = 0;
  _scalars["sum"]->setValue(0.0);
  _scalars["sumsquared"]->setValue(0.0);
  F0 = NF = 0; // nothing read yet

  N_AveReadBuf = 0;
  AveReadBuf = 0L;

  ReqF0 = 0;
  ReqNF = -1;
  Skip = 1;
  DoSkip = false;
  DoAve = false;
  _invalidCount = 0;
}


const QString& DataVector::typeString() const {
  return staticTypeString;
}


/** return true if it has a valid file and field, or false otherwise */
bool DataVector::isValid() const {
  if (dataSource()) {
    dataSource()->readLock();
    bool rc = dataSource()->vector().isValid(_field);
    dataSource()->unlock();
    return rc;
  }
  return false;
}


ScriptInterface* DataVector::createScriptInterface() {
  return new VectorDataSI(this);
}


bool DataVector::checkValidity(const DataSourcePtr& ds) const {
  if (ds) {
    ds->readLock();
    bool rc = ds->vector().isValid(_field);
    ds->unlock();
    return rc;
  }
  return false;
}

void DataVector::change(DataSourcePtr in_file, const QString &in_field,
                        int in_f0, int in_n,
                        int in_skip, bool in_DoSkip,
                        bool in_DoAve) {
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  Skip = in_skip;
  DoSkip = in_DoSkip;
  DoAve = in_DoAve;
  if (DoSkip && Skip < 1) {
    Skip = 1;
  }

  //_dontUseSkipAccel = false;
  setDataSource(in_file);
  ReqF0 = in_f0;
  ReqNF = in_n;
  _field = in_field;

  if (dataSource()) {
    dataSource()->writeLock();
  }
  reset();
  if (dataSource()) {
    dataSource()->unlock();
  }

  if (ReqNF <= 0 && ReqF0 < 0) {
    ReqF0 = 0;
  }
  registerChange();
}

qint64 DataVector::minInputSerial() const {
  if (dataSource()) {
    return (dataSource()->serial());
  }
  return LLONG_MAX;
}

qint64 DataVector::maxInputSerialOfLastChange() const {
  if (dataSource()) {
    return (dataSource()->serialOfLastChange());
  }
  return NoInputs;
}


void DataVector::changeFile(DataSourcePtr in_file) {
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  if (!in_file) {
    Debug::self()->log(tr("Data file for vector %1 was not opened.").arg(Name()), Debug::Warning);
  }
  setDataSource(in_file);
  if (dataSource()) {
    dataSource()->writeLock();
  }
  reset();
  if (dataSource()) {
    dataSource()->unlock();
  }
  registerChange();
}


void DataVector::changeFrames(int in_f0, int in_n,
                              int in_skip, bool in_DoSkip,
                              bool in_DoAve) {
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  if (dataSource()) {
    dataSource()->writeLock();
  }
  reset();
  if (dataSource()) {
    dataSource()->unlock();
  }
  Skip = in_skip;
  DoSkip = in_DoSkip;
  DoAve = in_DoAve;
  if (DoSkip && Skip < 1) {
    Skip = 1;
  }

  ReqF0 = in_f0;
  ReqNF = in_n;

  if (ReqNF <= 0 && ReqF0 < 0) {
    ReqF0 = 0;
  }
  registerChange();
}


void DataVector::setFromEnd() {
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  ReqF0 = -1;
  if (ReqNF < 2) {
    ReqNF = numFrames();
    if (ReqNF < 2) {
      ReqF0 = 0;
    }
  }
  registerChange();
}


DataVector::~DataVector() {
  if (AveReadBuf) {
    free(AveReadBuf);
    AveReadBuf = 0L;
  }
}


bool DataVector::readToEOF() const {
  return ReqNF <= 0;
}


bool DataVector::countFromEOF() const {
  return ReqF0 < 0;
}


/** Return Starting Frame of Vector */
int DataVector::startFrame() const {
  return F0;
}


/** Return frames per skip to read */
int DataVector::skip() const {
  return DoSkip ? Skip : 0;
}


bool DataVector::doSkip() const {
  return DoSkip;
}


bool DataVector::doAve() const {
  return DoAve;
}


/** Return frames held in Vector */
int DataVector::numFrames() const {
  return NF;
}


int DataVector::reqNumFrames() const {
  return ReqNF;
}


int DataVector::reqStartFrame() const {
  return ReqF0;
}


/** Save vector information */
void DataVector::save(QXmlStreamWriter &s) {
  if (dataSource()) {
    s.writeStartElement("datavector");
    saveFilename(s);
    s.writeAttribute("field", _field);

    s.writeAttribute("start", QString::number(ReqF0));
    s.writeAttribute("count", QString::number(ReqNF));

    if (doSkip()) {
      s.writeAttribute("skip", QString::number(Skip));
      if (doAve()) {
        s.writeAttribute("doAve", "true");
      }
    } else {
      s.writeAttribute("skip", QString::number(-1));
      s.writeAttribute("doAve", "false");
    }

    s.writeAttribute("startUnits", startUnits());
    s.writeAttribute("rangeUnits", rangeUnits());

    saveNameInfo(s, VNUM|XNUM);
    s.writeEndElement();
  }
}


/**
 * @brief Generate default label info for axis associated with this vector.
 * Use meta-scalars "units" or "quantity" if they are defined.
 * Escape special characters in the field name.
 *
 * @return LabelInfo
 **/

LabelInfo DataVector::labelInfo() const {
  LabelInfo label_info;

  if (_fieldStrings.contains("quantity")) {
    label_info.quantity = _fieldStrings.value("quantity")->value();
    label_info.quantity.replace('[', "\\[").replace(']', "\\]");
  } else {
    label_info.quantity.clear();
  }

  if (_fieldStrings.contains("units")) {
    label_info.units = _fieldStrings.value("units")->value();
    label_info.units.replace('[', "\\[").replace(']', "\\]");
  } else {
    label_info.units.clear();
  }

  label_info.name = descriptiveName();// _field;

  label_info.file = filename();

  return label_info;
}


void DataVector::reset() { // must be called with a lock
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  //_dontUseSkipAccel = false;
  if (dataSource()) {
    SPF = dataInfo(_field).samplesPerFrame;
  }
  F0 = NF = 0;
  resize(0);
  _numSamples = 0;
  _dirty = true;
  _resetFieldMetadata();

  Object::reset();
}


bool DataVector::checkIntegrity() {
  if (DoSkip && Skip < 1) {
    Skip = 1;
  }

  if (_dirty) {
    reset();
  }

  // if the file seems to have shrunk/changed, return false.
  // if it has happened several times in a row, assume the
  // file has been over-written, and re-read it.
  // this is a hack to handle glitchy file system situations.
  // TODO: there has to be a better way.
  const DataInfo info = dataInfo(_field);
  if (dataSource() && (SPF != info.samplesPerFrame || info.frameCount < NF)) {
    _invalidCount++;
    if (_invalidCount>INVALIDS_PER_RESET) {

      reset();
      _invalidCount=0;
    }
    return false;
  }

  // check for illegal NF and F0 values
  if (ReqNF < 1 && ReqF0 < 0) {
    ReqF0 = 0; // for this illegal case, read the whole file
  }

  if (ReqNF == 1) {
    ReqNF = 2;
  }

  _invalidCount = 0;
  return true;
}

// Some things to consider about the following routine...
// Frames:
//    Some data sources have data divided into frames.  Each field
//    has a fixed number of samples per frame.  For some (eg, ascii files)
//    each frame has 1 sample.  For others (eg, dirfiles) you may have more.
//    Different fields in the same data source may have different samples per frame.
//    Within a data source, it is assumed that the first sample of each frame is
//    simultaneous between fields.
// Last Frame Read:
//    Only read the first sample of the last frame read, in cases where there are more
//    than one sample per frame.   This allows for more sensible association of vectors
//    into curves, when the X and Y vectors have different numbers of samples per frame.
//    The rule is that we assume is that the first sample of each frame is simultaneous.
// Skip reading:  
//    -'Skip' means read 1 sample each 'Skip' frames (not read one sample,
//     then skip 'Skip' samples or something else).
//    -In order that the data are not re-drawn each time a new sample arrives, and to
//     ensure the re-usability (via shifting) of previously read data, and to make a
//     region of data look the same regardless of the chouse of f0, all samples
//     read with skip enabled are read on 'skip boundries'... ie, the first samples of
//     frame 0, Skip, 2*Skip... N*skip, and never M*Skip+1.

void DataVector::internalUpdate() {
  int i, k, shift, n_read=0;
  int ave_nread;
  int new_f0, new_nf;
  bool start_past_eof = false;

  if (dataSource()) {
    dataSource()->writeLock();
  } else {
    return;
  }

  const DataInfo info = dataInfo(_field);
  if (!checkIntegrity()) {
    if (dataSource()) {
      dataSource()->unlock();
    }
    return;
  }

  if (DoSkip && Skip < 2 && SPF == 1) {
    DoSkip = false;
  }


  // set new_nf and new_f0
  int fc = info.frameCount;
  if (ReqNF < 1) { // read to end of file
    new_f0 = ReqF0;
    new_nf = fc - new_f0;
  } else if (ReqF0 < 0) { // count back from end of file
    new_nf = fc;
    if (new_nf > ReqNF) {
      new_nf = ReqNF;
    }
    new_f0 = fc - new_nf;
  } else {
    new_f0 = ReqF0;
    new_nf = ReqNF;
    if (new_f0 + new_nf > fc) {
      new_nf = fc - new_f0;
    }
    if (new_nf <= 0) {
      // Tried to read starting past the end.
      new_f0 = 0;
      new_nf = 1;
      start_past_eof = true;
    }
  }

  if (DoSkip) {
    // in count from end mode, change new_f0 and new_nf so they both lie on skip boundaries
    if ((new_f0 != 0) && (ReqF0<0)) {
      new_f0 = ((new_f0-1)/Skip+1)*Skip;
    }
    new_nf = (new_nf/Skip)*Skip;
  }

  // shift vector if necessary
  if (new_f0 < F0 || new_f0 >= F0 + NF) { // No useful data around.
    reset();
  } else { // shift stuff rather than re-read
    if (DoSkip) {
      shift = (new_f0 - F0)/Skip;
      NF -= (new_f0 - F0);
      _numSamples = NF/Skip;
    } else {
      shift = SPF*(new_f0 - F0);
      NF -= (new_f0 - F0);
      //_numSamples = (NF-1)*SPF;
      if (shift > 0) {
        if (SPF == 1) {
          _numSamples = NF;
        } else {
          _numSamples = (NF-1)*SPF;
        }
      }
    }

    memmove(_v, _v+shift, _numSamples*sizeof(double));
  }

  if (DoSkip) {
    // reallocate V if necessary
    if (new_nf / Skip != _size) {
      if (! resize(new_nf/Skip)) {
        // TODO: Is aborting all we can do?
        fatalError("Not enough memory for vector data");
        return;
      }
    }
    // FIXME maybe.
    //   -skip acceleration is not supported by any current data sources.
    //   -there is no API implemented to report if a data source implements skip accel
    // so: for now, just say it isn't supported here.  Fix if we ever implement a data source
    //     where skip accel is important (eg, planck HFI...)
#if 0
    _dontUseSkipAccel = true;
    if (!_dontUseSkipAccel) { // ie, use skip accel.  Not used.
      int rc;
      int lastRead = -1;
      if (DoAve) {
        // We don't support boxcar inside data sources yet.
        _dontUseSkipAccel = true;
      } else {
        rc = readField(_v + _numSamples, _field, new_f0, (new_nf - NF)/Skip, Skip, &lastRead);
        if (rc != -9999) {
          if (rc >= 0) {
            n_read = rc;
          } else {
            n_read = 0;
          }
        } else {
          _dontUseSkipAccel = true;
        }
      }
    }
#endif
//  if (_dontUseSkipAccel) {
    n_read = 0;
    /** read each sample from the File */
    double *t = _v + _numSamples;
    int new_nf_Skip = new_nf - Skip;
    if (DoAve) {
      for (i = NF; new_nf_Skip >= i; i += Skip) {
        /* enlarge AveReadBuf if necessary */
        if (N_AveReadBuf < Skip*SPF) {
          N_AveReadBuf = Skip*SPF;
          if (!kstrealloc(AveReadBuf, N_AveReadBuf*sizeof(double))) {
            qCritical() << "Vector resize failed";
          }
          if (!AveReadBuf) {
            // FIXME: handle failed resize
          }
        }
        ave_nread = readField(AveReadBuf, _field, new_f0+i, Skip);
        for (k = 1; k < ave_nread; ++k) {
          AveReadBuf[0] += AveReadBuf[k];
        }
        if (ave_nread > 0) {
          *t = AveReadBuf[0]/double(ave_nread);
          n_read++;
        }
        ++t;
      }
    } else {
      for (i = NF; new_nf_Skip >= i; i += Skip) {
        n_read += readField(t++, _field, new_f0 + i, -1);
      }
    }
//  } // end if _dontUseSkipAccel
  } else {
    // reallocate V if necessary
    if ((new_nf - 1)*SPF + 1 != _size) {
      if (!resize((new_nf - 1)*SPF + 1)) {
        // TODO: Is aborting all we can do?
        fatalError("Not enough memory for vector data");
        return;
      }
    }

    //if (NF > 0) {
    //  NF--; /* last frame read was only partially read... */
    //}

    // read the new data from file
    if (start_past_eof) {
      _v[0] = NOPOINT;
      n_read = 1;
    } else if (info.samplesPerFrame > 1) {
      if (NF>0) {
        NF--;  /* last frame read was only partially read... */
      }
      int safe_nf = (new_nf>0 ? new_nf : 0);

      assert(new_f0 + NF >= 0);
      //assert(new_f0 + safe_nf - 1 >= 0);
      if ((new_f0 + safe_nf - 1 >= 0) && (safe_nf - NF > 1)) {
        n_read = readField(_v+NF*SPF, _field, new_f0 + NF, safe_nf - NF - 1);
        n_read += readField(_v+(safe_nf-1)*SPF, _field, new_f0 + safe_nf - 1, -1);
      }
    } else {
      assert(new_f0 + NF >= 0);
      if (new_nf - NF > 0 || new_nf - NF == -1) {
        n_read = readField(_v+NF*SPF, _field, new_f0 + NF, new_nf - NF);
      }
    }
  }
  NumNew = _size - _numSamples;
  NF = new_nf;
  F0 = new_f0;
  _numSamples += n_read;

  // if for some reason (eg, realtime reading an nfs mounted
  // dirfile) not all of the data was read, the data will never
  // be read; the samples will be filled in with the last data
  // point read, and a complete reset will be requested.
  // This is bad - I think it will be worthwhile
  // to add blocking w/ timeout to KstFile.
  // As a first fix, mount all nsf mounted dirfiles with "-o noac"
  _dirty = false;
  if (_numSamples != _size && !(_numSamples == 0 && _size == 1)) {
    _dirty = true;
    for (i = _numSamples; i < _size; ++i) {
      _v[i] = _v[0];
    }
  }

  if (NumNew > _size) {
    NumNew = _size;
  }
  if (NumShifted > _size) {
    NumShifted = _size;
  }

  if (dataSource()) {
    dataSource()->unlock();
  }

  if (n_read>0) {
    Vector::internalUpdate();
  }
}

QByteArray DataVector::scriptInterface(QList<QByteArray> &c)
{
    Q_ASSERT(c.size());
    if(c[0]=="changeFrames") {
//        int f0, int n, int skip, bool in_doSkip, bool in_doAve;
        if(c.size()!=6) {
            return QByteArray("Bad parameter count (!=5).");
        }
        changeFrames(c[1].toInt(),c[2].toInt(),c[3].toInt(),(c[4]=="true")?1:0,(c[5]=="true")?1:0);
        return QByteArray("Done.");
    } else if(c[0]=="numFrames") {
        return QByteArray::number(numFrames());
    } else if(c[0]=="startFrame") {
        return QByteArray::number(startFrame());
    } else if(c[0]=="doSkip") {
        return QByteArray(doSkip()?"true":"false");
    } else if(c[0]=="doAve") {
        return QByteArray(doAve()?"true":"false");
    } else if(c[0]=="skip") {
        return QByteArray::number(skip());
    } else if(c[0]=="reload") {
        reload();
        return QByteArray("Done");
    } else if(c[0]=="samplesPerFrame") {
        return QByteArray::number(samplesPerFrame());
    } else if(c[0]=="fileLength") {
        return QByteArray::number(fileLength());
    } else if(c[0]=="readToEOF") {
        return QByteArray(readToEOF()?"true":"false");
    } else if(c[0]=="countFromEOF") {
        return QByteArray(countFromEOF()?"true":"false");
    } else if(c[0]=="descriptionTip") {
        return QByteArray(descriptionTip().toLatin1());
    } else if(c[0]=="isValid") {
        return isValid()?"true":"false";
    }

    return "No such command...";
}


/** Returns intrinsic samples per frame */
int DataVector::samplesPerFrame() const {
  return SPF;
}


int DataVector::fileLength() const {

  if (dataSource()) {
    int rc = dataInfo(_field).frameCount;

    return rc;
  }

  return 0;
}


void DataVector::reload() {
  Q_ASSERT(myLockStatus() == KstRWLock::WRITELOCKED);

  if (dataSource()) {
    dataSource()->writeLock();
    dataSource()->reset();
    dataSource()->unlock();
    reset();
    _resetFieldMetadata();
    registerChange();
  }
}

void DataVector::_resetFieldMetadata() {
  _resetFieldScalars();
  _resetFieldStrings();
}

void DataVector::_resetFieldStrings() {
  const QMap<QString, QString> meta_strings = dataSource()->vector().metaStrings(_field);
  
  QStringList fieldStringKeys = _fieldStrings.keys();
  // remove field strings that no longer need to exist
  readLock();
  for (int i=0; i<fieldStringKeys.count(); ++i) {
    QString key = fieldStringKeys.at(i);
    if (!meta_strings.contains(key)) {
      StringPtr sp = _fieldStrings[key];
      _strings.remove(key);
      _fieldStrings.remove(key);
      sp = 0L;
    }
  }
  // find or insert strings, to set their value
  QMapIterator<QString, QString> it(meta_strings);
  while (it.hasNext()) {
    it.next();
    QString key = it.key();
    StringPtr sp;
    if (!_fieldStrings.contains(key)) { // insert a new one
      _strings.insert(key, sp = store()->createObject<String>());
      _fieldStrings.insert(key, sp);
      sp->setProvider(this);
      sp->setSlaveName(key);
    } else {  // find it
      sp = _fieldStrings[key];
    }
    sp->setValue(it.value());
  }
  unlock();
}


void DataVector::_resetFieldScalars() {
  const QMap<QString, double> meta_scalars = dataSource()->vector().metaScalars(_field);


  QStringList fieldScalarKeys = _fieldScalars.keys();
  // remove field scalars that no longer need to exist
  readLock();
  for (int i=0; i<fieldScalarKeys.count(); ++i) {
    QString key = fieldScalarKeys.at(i);
    if (!meta_scalars.contains(key)) {
      ScalarPtr sp = _fieldScalars[key];
      _scalars.remove(key);
      _fieldScalars.remove(key);
      sp = 0L;
    }
  }
  // find or insert scalars, to set their value
  QMapIterator<QString, double> it(meta_scalars);
  while (it.hasNext()) {
    it.next();
    QString key = it.key();
    ScalarPtr sp;
    if (!_fieldScalars.contains(key)) { // insert a new one
      _scalars.insert(key, sp = store()->createObject<Scalar>());
      _fieldScalars.insert(key, sp);
      sp->setProvider(this);
      sp->setSlaveName(key);
    } else {  // find it
      sp = _fieldScalars[key];
    }
    sp->setValue(it.value());
  }
  unlock();
}


PrimitivePtr DataVector::makeDuplicate() const {
  Q_ASSERT(store());
  DataVectorPtr vector = store()->createObject<DataVector>();

  vector->writeLock();
  vector->change(dataSource(), _field, ReqF0, ReqNF, Skip, DoSkip, DoAve);
  if (descriptiveNameIsManual()) {
    vector->setDescriptiveName(descriptiveName());
  }

  vector->registerChange();
  vector->unlock();

  return kst_cast<Primitive>(vector);
}

QString DataVector::_automaticDescriptiveName() const {
  QString name;
  name = _field;
  // un-escape escaped special characters so they aren't escaped 2x.
  name.replace("\\_", "_").replace("\\^","^").replace("\\[", "[").replace("\\]", "]");
  // now escape the special characters.
  name.replace('_', "\\_").replace('^', "\\^").replace('[', "\\[").replace(']', "\\]");
  return name;
}

QString DataVector::descriptionTip() const {
  QString IDstring;
  //QString range_string;

  IDstring = tr(
      "Data Vector: %1\n"
      "  %2\n"
      "  Field: %3"
      ).arg(Name()).arg(dataSource()->fileName()).arg(_field);

  if (countFromEOF()) {
    IDstring += tr("\n  Last %1 frames.").arg(numFrames());
  } else if (readToEOF()) {
    IDstring += tr("\n  Frame %1 to end.").arg(startFrame());
  } else {
    IDstring += tr("\n  %1 Frames starting at %2.").arg(numFrames()).arg(startFrame());
  }
  if (skip()) {
    if (!doAve()) {
      IDstring+=tr("\n  Read 1 sample per %1 frames.").arg(skip());
    } else {
      IDstring+=tr("\n  Average each %1 frames.").arg(skip());
    }
  }
  return IDstring;
}

QString DataVector::propertyString() const {
  return tr("%2 F0: %3 N: %4 of %1", "%2 is a variable name.  F0 is short for the first element.  N is the number of elements").arg(dataSource()->fileName()).arg(_field).arg(startFrame()).arg(numFrames());
}


int DataVector::readField(double *v, const QString& field, int s, int n, int skip)
{
  ReadInfo par = {v, s, n, skip};
  return dataSource()->vector().read(field, par);
}

const DataVector::DataInfo DataVector::dataInfo(const QString& field) const
{
  dataSource()->readLock();
  const DataInfo info = dataSource()->vector().dataInfo(field);
  dataSource()->unlock();
  return info;
}

bool DataVector::isTime() const {
  return dataSource()->isTime(_field);
}

}
// vim: ts=2 sw=2 et
