!-----------------------------------------------------------------------
! Offline data processing. Available modes:
!  /DAILY: use a circular index with yesterday and today scans
!  /DATES FirstDate LastDate: process this range of dates
! One (and only one) mode is mandatory
!-----------------------------------------------------------------------
!
begin procedure pipeline-offline-init
  !---------------------------------------------------------------------
  ! Set the tuning structure
  !---------------------------------------------------------------------
  !
  @ pipeline-structure
  !
  let pipe%imbfits       "/ncsServer/mrt/ncs/data/imbfits/het/"
  let pipe%indexdir      "./raw-mrtcal/"
  let pipe%classdir      "."  ! Unused
  let pipe%projfits      " "
  let pipe%docont        yes
  let pipe%plot%do       no
  let pipe%plot%pdir     " "  ! Unused
  let pipe%plot%odir     " "  ! Unused
  let pipe%plot%osubdir  " "  ! Unused
  let pipe%plot%osymlink " "  ! Unused
  let pipe%plot%error    "return"  ! Unused
  let pipe%vodir         " "  ! Unused
  let pipe%stopfile      " "  ! Unused
  let pipe%simulator     no
  let pipe%simudate      " "  ! Unused
  let pipe%dynaproc      " "  ! Unused
  let pipe%info%status   " "  ! Unused
  let pipe%info%log      " "  ! Unused
  let pipe%logfile       " "  ! Not relevant
  !
  ! MSET tunings
  mset  bookkeeping  space        1024
  !
  mset  calibration  bad          no
  mset  calibration  bandwidth    0.0  ! Play fast
  mset  calibration  chopper      strict
  mset  calibration  feedback     set
  mset  calibration  interval     30.0
  mset  calibration  match        0.040
  mset  calibration  mjdinter     yes
  mset  calibration  off          linear  ! For OTF PSW
  mset  calibration  products     linear
  mset  calibration  scan         nearest
  mset  calibration  water        element  ! 1 pwv per calib bandwidth
  !
  mset  output       calibration  associated
  mset  output       caltable     no
  mset  output       chunk        set
  mset  output       integration  *
  mset  output       spectra      no  ! Play fast
  mset  output       usersection  no
  mset  output       vodir        "."  ! Unused
  mset  output       voxml        no
  mset  output       weight       no
  !
  ! Set up environment
  sic mkdir 'pipe%indexdir'
  !
end procedure pipeline-offline-init
!
begin procedure pipeline-offline-daily
  !---------------------------------------------------------------------
  ! Daily mode: this scheme uses a single index.mrt which contains only
  ! 1 or 2 days of files indexed.
  ! 1) Strip old dates and old versions from the index file, i.e. keep
  !    only the "latest date - 1" and "latest date" in the index. This
  !    step is done only if a previous index exists!
  ! 2) Update the index file up to today (included), i.e. index the
  !    IMB-FITS files which are not yet indexed.
  ! 3) Calibrate what is not yet calibrated.
  !---------------------------------------------------------------------
  !
  define character*256 index
  let index 'pipe%indexdir'"/index.mrt"
  !
  define character*11 lastdate prevdate
  ! Default if no index file yet, or if index file is empty:
  let lastdate "yesterday"
  let prevdate "yesterday"
  !
  if file(index) then
    ! Find the latest day in the index.mrt
    index open 'pipe%imbfits' /file 'index'
    mfind
    ml /toc observed
    !
    if (mtoc%nsetup.gt.0) then
      ! Index file is not empty
      let lastdate 'mtoc%obs[mtoc%nobs]'  ! Assume last date is latest OBSERVED value
      !
      ! Compute "latest day - 1". It is trivial if done through gag_dates.
      define integer gagdate
      compute gagdate gag_date 'lastdate'
      let gagdate gagdate-1
      compute prevdate gag_date gagdate
      !
      ! Keep only entries from "latest day - 1" (so that we can find
      ! calibrations on the previous day) to "today"
      say ""
      message i pipeline "Stripping index entries older than "'prevdate'
      ! Keep only 'prevdate' to today entries
      index open 'pipe%imbfits' /file 'index'
      mfind /date 'prevdate' to today
      mcopy 'pipe%indexdir'"/new-index.mrt"  ! Can not overwrite the index file, use intermediate
      sic rename 'pipe%indexdir'"/new-index.mrt" 'index'  ! Now overwrite the old index
      !
      ! Force loading the new file and recomputing IX
      index open 'pipe%imbfits' /file 'index'
      !
    endif
  endif
  !
  ! Update (or create if needed) the index file, starting from
  ! latest date currently in index, up to now
  index update 'pipe%imbfits' /file 'index' /date 'lastdate' to today
  !
  ! Calibrate, starting from the day before latest date (there may
  ! be files indexed before midnight but not yet calibrated)
  @ pipeline-process 'prevdate' today
  !
end procedure pipeline-offline-daily
!
begin procedure pipeline-offline-dates
  !---------------------------------------------------------------------
  ! Reprocess a range of dates:
  !  pro%arg[1]: first date, format YYYYMMDD
  !  pro%arg[2]: last date, format YYYYMMDD
  !---------------------------------------------------------------------
  !
  define char*8 fdate ldate
  let fdate 'pro%arg[1]'
  let ldate 'pro%arg[2]'
  !
  define character*256 index
  let index 'pipe%indexdir'"/"'fdate'"-"'ldate'".mrt"
  !
  ! (Re)build the index file
  index update 'pipe%imbfits' /file 'index' /date 'fdate' to 'ldate'
  !
  ! Calibrate
  @ pipeline-process 'fdate' 'ldate'
  !
end procedure pipeline-offline-dates
!
@ pipeline-tools
@ pipeline-offline-init

! Command line parsing
sic\parse 0 /daily 0 /dates 2

if pro%parse%daily%narg.ge.0 then
  @ pipeline-offline-daily
else if pro%parse%dates%narg.ge.0 then
  @ pipeline-offline-dates 'pro%parse%dates%arg[1]' 'pro%parse%dates%arg[2]'
else
  message w pipeline-offline "No mode given, nothing done"
endif

exit  ! Exit the whole program
