!-----------------------------------------------------------------------
! Online data processing procedure. Calibrate new (uncalibrated) files
! from yesterday and today. Files already present on disk are indexed
! and calibrated first, then all new files are also indexed and
! calibrated as they appear.
!
! Calling sequence:
! @ pipeline-odp  [/SIMULATOR] [/SKIP]
!
!  /SIMULATOR: enable the simulator mode
!  /SKIP: index new files from yesterday and today up to now, but mark
!         them as SKIPPED instead of calibrating them. The new files
!         appearing after this step will be calibrated as they appear.
!  /LOG FileName: name of the log file for this session. In order to
!         avoid troubles when parsing the options because of leading
!         / in absolute paths (e.g. /LOG /path/to/log), the log file
!         is assumed to be prefixed with "file:"
!-----------------------------------------------------------------------
begin procedure pipeline-odp-init-online
  !---------------------------------------------------------------------
  ! Set up the variables and environment for online OdP calibration
  ! pro%arg[1]  log file name
  !---------------------------------------------------------------------
  !
  @ pipeline-structure
  @ pipeline-odp-env-online 'pro%arg[1]'
  @ pipeline-odp-setup
  @ pipeline-info-logfile
  !
end procedure pipeline-odp-init-online
!
begin procedure pipeline-odp-env-online
  !---------------------------------------------------------------------
  ! Set up the environment for online observations
  ! (static variables only, i.e. variables which do not change during
  ! the pipeline process).
  ! pro%arg[1]  log file name
  !---------------------------------------------------------------------
  !
  let pipe%imbfits       "/ncsServer/mrt/ncs/data/imbfits/het/"
  let pipe%indexdir      "./index/"
  let pipe%stopfile      "./mrtcal.stop"
  let pipe%docont        no
  let pipe%simulator     no
  let pipe%simudate      " "  ! Unused
  let pipe%plot%do       yes
  let pipe%plot%odir     "/ncsServer/mrt/ncs/monitor/plots"
  let pipe%plot%osubdir  "mrtcal"
  let pipe%plot%osymlink "lastScan.html"
  let pipe%plot%error    "return"
  let pipe%vodir         "/ncsServer/mrt/ncs/data"  ! XML will go in /ncsServer/mrt/ncs/data/YYYYMMDD/scans/NN/
  let pipe%logfile       'pro%arg[1]'
  !
  ! Other variables handled here:
  let pipe%dynaproc  "pipeline-odp-env-online-dyna"
  !
  ! Info files. Must be in locations easy to find by odp.sh
  define char*128 infodir
  let infodir  "/tmp/mrtcal/"'sic%user'
  sic mkdir 'infodir'
  let pipe%info%status 'infodir'"/status.nfo"
  let pipe%info%log    'infodir'"/logfile.nfo"
  !
end procedure pipeline-odp-env-online
!
begin procedure pipeline-odp-env-online-dyna
  !---------------------------------------------------------------------
  ! Set up the environment for online observations
  ! (dynamic variables only, i.e. variables which may change during the
  ! pipeline reduction process, reevaluated each time needed, depending
  ! on the input parameters).
  !
  !  QUEUE and PROJECT values are read from first IMB-FITS in index.
  ! This assumes the index contains a single QUEUE and PROJECT
  ! combination. This assumption *may* not be true because we iterate
  ! (MFIND) scans by projid and not also by queue. Note however that
  ! I hardly see how we can observe the same projid but not with the
  ! same queue!
  !---------------------------------------------------------------------
  !
  m\mset input data none  ! Data not needed, do not waste time
  m\read first
  m\variable imbf         ! Only PRIMARY HDU needed
  !
  ! Keep memory of the current queue in the global structure
  let pipe%queue 'imbf%prim%queue'
  !
  ! Target directory? Heuristic depend if pool or not pool
  define char*12 projid queue
  let projid 'imbf%prim%projid'
  let queue 'imbf%prim%queue'
  !
  define character*256 projdir queuedir
  let projdir   "/mrt-lx3/vis/"'projid'
  let queuedir  "/mrt-lx3/vis/"'queue'
  !
  define logical dofallback
  let dofallback no
  !
  if imbf%prim%queue[1:4].eq."pool" then  ! Pool
    if file(projdir) then
      let pipe%classdir  'projdir'"/observationData/mrtcal"
      let pipe%projfits  'projdir'"/observationData/imbfits"
      let pipe%plot%pdir 'projdir'"/observationData/mrtcal/plots"
    else if file(queuedir) then
      let pipe%classdir  'queuedir'"/observationData/"'projid'"/mrtcal/"
      let pipe%projfits  'queuedir'"/observationData/"'projid'"/imbfits"
      let pipe%plot%pdir 'queuedir'"/observationData/"'projid'"/mrtcal/plots"
    else
      let dofallback yes
    endif
    !
  else  ! Not pool
    if file(queuedir) then
      if imbf%prim%projid.eq.imbf%prim%queue then
        ! This is normal observing mode
        let pipe%classdir  'queuedir'"/observationData/mrtcal"
        let pipe%projfits  'queuedir'"/observationData/imbfits"
        let pipe%plot%pdir 'queuedir'"/observationData/mrtcal/plots"
      else
        ! e.g. SET PROJECT TEST executed under a project account
        let pipe%classdir  'queuedir'"/observationData/"'projid'"/mrtcal/"
        let pipe%projfits  'queuedir'"/observationData/"'projid'"/imbfits"
        let pipe%plot%pdir 'queuedir'"/observationData/"'projid'"/mrtcal/plots"
      endif
    else
      let dofallback yes
    endif
  endif
  !
  ! Last chance fallback
  if dofallback then
    define char*12 fallback
    datetime /from now /to fallback YYYYMMDD
    let fallback "000-"'fallback[3:4]'  ! e.g. 000-17
    !
    define character*256 falldir
    let falldir "/mrt-lx3/vis/"'fallback'
    !
    let pipe%classdir  'falldir'"/observationData/"'queue'"-"'projid'"/mrtcal/"
    let pipe%projfits  'falldir'"/observationData/"'queue'"-"'projid'"/imbfits"
    let pipe%plot%pdir 'falldir'"/observationData/"'queue'"-"'projid'"/mrtcal/plots"
  endif
  !
  ! These ones is dynamic, create if needed each time
  if (.not.file(pipe%projfits)) then
    message i pipeline "Creating index directory "'pipe%projfits'
    sic mkdir 'pipe%projfits'
  endif
  if (.not.file(pipe%classdir)) then
    message i pipeline "Creating .30m directory "'pipe%classdir'
    sic mkdir 'pipe%classdir'
  endif
  if (.not.file(pipe%plot%pdir)) then
    message i pipeline "Creating plots directory "'pipe%plot%pdir'
    sic mkdir 'pipe%plot%pdir'
  endif
  !
end procedure pipeline-odp-env-online-dyna
!
begin procedure pipeline-odp-init-simulator
  !---------------------------------------------------------------------
  ! Set up the variables and environment for OdP calibration using
  ! simulator mode.
  ! pro%arg[1]  log file name
  !---------------------------------------------------------------------
  !
  message i pipeline ">>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<"
  message i pipeline ">>>          Entering simulator mode        <<<"
  message i pipeline ">>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<"
  !
  @ pipeline-structure
  @ pipeline-odp-env-simu 'pro%arg[1]'
  @ pipeline-odp-setup
  @ pipeline-info-logfile
  !
end procedure pipeline-odp-init-simulator
!
begin procedure pipeline-odp-env-simu
  !---------------------------------------------------------------------
  ! Set up the environment for OdP calibration using simulator mode
  ! (static variables only, i.e. variables which do not change during
  ! the pipeline process).
  ! pro%arg[1]  log file name
  !---------------------------------------------------------------------
  !
  let pipe%imbfits       "./imbfits"  ! The watching directory
  let pipe%indexdir      "./index"
  let pipe%classdir      "./mrtcal"
  let pipe%docont        no  ! or yes in simulator mode?
  let pipe%plot%do       yes
  let pipe%plot%pdir     "./plots"
  let pipe%plot%odir     "./oplots"
  let pipe%plot%osubdir  "mrtcal"
  let pipe%plot%osymlink "lastScan.html"
  let pipe%plot%error    "return"
  let pipe%vodir         "./VOXML"
  let pipe%stopfile      "./mrtcal.stop"
  let pipe%simulator     yes
  let pipe%simudate      "./imbfits/pipeline-simulator.datetime"
  let pipe%logfile       'pro%arg[1]'
  !
  ! Other variables handled here:
  let pipe%dynaproc  "pipeline-odp-env-simu-dyna"
  !
  ! Info files. Must be in locations easy to find by odp.sh
  define char*128 infodir
  let infodir  "/tmp/mrtcal/"'sic%user'
  sic mkdir 'infodir'
  let pipe%info%status 'infodir'"/status.nfo"
  let pipe%info%log    'infodir'"/logfile.nfo"
  !
  ! These ones are static is this context:
  if .not.file(pipe%classdir) then
    message i pipeline "Creating .30m directory "'pipe%classdir'
    sic mkdir 'pipe%classdir'
  endif
  if .not.file(pipe%plot%pdir) then
    message i pipeline "Creating plots directory "'pipe%plot%pdir'
    sic mkdir 'pipe%plot%pdir'
  endif
  !
end procedure pipeline-odp-env-simu
!
begin procedure pipeline-odp-env-simu-dyna
  !---------------------------------------------------------------------
  ! Set up the environment for OdP calibration using simulator mode
  ! (dynamic variables only, i.e. variables which may change during the
  ! pipeline reduction process, reevaluated each time needed, depending
  ! on the input parameters).
  !
  !  QUEUE and PROJECT values are read from first IMB-FITS in index.
  ! This assumes the index contains a single QUEUE and PROJECT
  ! combination. This assumption *may* not be true because we iterate
  ! (MFIND) scans by projid and not also by queue. Note however that
  ! I hardly see how we can observe the same projid but not with the
  ! same queue!
  !---------------------------------------------------------------------
  !
  m\mset input data none  ! Data not needed, do not waste time
  m\read first
  m\variable imbf         ! Only PRIMARY HDU needed
  !
  ! Keep memory of the current queue in the global structure
  let pipe%queue 'imbf%prim%queue'
  !
  ! Target directory for per-project per-day index.
  ! - Can not be equal to pipe%imbfits as this procedure would create a
  !   new file in the watching directory, messing up the watching
  !   process itself.
  ! - Can not be equal to pipe%indexdir as these main indexes and
  !   per-project indexes would conflict.
  let pipe%projfits  "./index/"'imbf%prim%projid'
  if (.not.file(pipe%projfits)) then
    message i pipeline "Creating per project index directory "'pipe%projfits'
    sic mkdir 'pipe%projfits'
  endif
  !
end procedure pipeline-odp-env-simu-dyna
!
begin procedure pipeline-odp-setup
  !
  ! MSET tunings
  mset  bookkeeping  space        1024
  !
  mset  calibration  bad          no
  mset  calibration  bandwidth    20.0
  mset  calibration  chopper      strict
  mset  calibration  feedback     set
  mset  calibration  interval     30.0
  mset  calibration  match        0.040
  mset  calibration  mjdinter     yes
  mset  calibration  off          linear  ! For OTF PSW
  mset  calibration  products     linear
  mset  calibration  scan         nearest
  mset  calibration  water        element  ! 1 pwv per calib bandwidth
  !
  mset  output       calibration  associated
  mset  output       caltable     yes
  mset  output       chunk        set
  mset  output       integration  *
  mset  output       spectra      yes
  mset  output       usersection  no
  mset  output       vodir        'pipe%vodir'
  mset  output       voxml        yes
  mset  output       weight       no
  !
  ! Set up environment
  sic mkdir 'pipe%indexdir'
  if pipe%plot%odir.ne." " then
    message i pipeline "Creating plots directory "'pipe%plot%odir'
    sic mkdir 'pipe%plot%odir'
    sic mkdir 'pipe%plot%odir'"/"'pipe%plot%osubdir'
  endif
  !
  ! Ensure the exit trigger from previous session is removed:
  sic delete 'pipe%stopfile'
  !
end procedure pipeline-odp-setup
!
begin procedure pipeline-odp-process
  !---------------------------------------------------------------------
  ! By-date scheme. This creates/updates one index file per date.
  ! Note that this will not index nor calibrate the files older than
  ! yesterday.
  !
  ! Calling sequence:
  !  @ pipeline-odp-process [/SKIP]
  !
  ! /SKIP: skip data (i.e. index them but do not calibrate them, mark
  !        as SKIPPED) previously acquired for yesterday and today up to
  !        now.
  !---------------------------------------------------------------------
  !
  parse 0 /skip 0
  define logical skip
  let skip pro%parse%skip%narg.ge.0
  !
  define char*8 yesterday today
  if pipe%simulator then
    ! Simulator mode
    define double nowmjd
    if .not.file(pipe%simudate) then
      message w pipeline "File "'pipe%simudate'" not found, assuming no simulated files present yet"
      return
    endif
    accept nowmjd /column 'pipe%simudate'
    datetime /from nowmjd MJD /to today     YYYYMMDD
    let nowmjd nowmjd-1
    datetime /from nowmjd MJD /to yesterday YYYYMMDD
  else
    ! Online mode
    datetime /from previous DAY /to yesterday YYYYMMDD
    datetime /from NOW          /to today     YYYYMMDD
    ! NB: in the code below we use the variables 'yesterday' and 'today',
    ! and not the keywords YESTERDAY and TODAY, so that we remain
    ! consistant during the procedure execution, even if we overlap
    ! midnight at this precise moment.
  endif
  !
  define char*256 yindex tindex
  let yindex 'pipe%indexdir'"/"'yesterday'
  let tindex 'pipe%indexdir'"/"'today'
  !
  ! Create/update 2 separate files
  say ""
  message i pipeline "Updating main index files for yesterday and today"
  index update 'pipe%imbfits' /file 'yindex' /date 'yesterday'
  index update 'pipe%imbfits' /file 'tindex' /date 'today'
  !
  ! Open these 2 files in memory
  index open   'pipe%imbfits' /file 'yindex'
  index append 'pipe%imbfits' /file 'tindex'
  !
  if skip then
    ! Skip what is not yet calibrated in these files
    @ pipeline-skip 'yesterday' 'today'
  else
    ! Calibrate what is not yet calibrated in these files
    @ pipeline-process 'yesterday' 'today'
  endif
  !
end procedure pipeline-odp-process
!
begin procedure pipeline-odp-watch
  !---------------------------------------------------------------------
  ! Wait for new data and calibrate this data, and wait again and so on.
  ! Infinite loop.
  !---------------------------------------------------------------------
  !
  define char*8 today
  sic modified 'pipe%imbfits' pipe%modif
  !
  for /while .true.
    say ""
    sic date
    message i pipeline "Waiting for new data since "'sys_date'" ..."
    @ pipeline-info-idle
    !
    ! Infinite wait (no time-out). The commands below are carefully
    ! chosen to deal correctly with midnight. Note that INDEX WATCH
    ! is NOT suited to deal correctly with 2 input indexes and
    ! updating one of them (which one?) with /DATE TODAY, especially
    ! when midnight occurs...
    for /while .true.
      if file(pipe%stopfile) then
        message i pipeline "File "'pipe%stopfile'" found in current directory => exiting"
        exit
      endif
      sic wait 1.0
      sic modified 'pipe%imbfits' pipe%modif
      if pipe%modif%modif  break
    next
    !
    ! Update yesterday (useful if overlapping midnight), today,
    ! and calibrate new files from those 2 indices.
    @ pipeline-odp-process
    !
  next
  !
end procedure pipeline-odp-watch
!
message i pipeline "Running as "'sic%user'"@"'sic%host'
say ""
!
parse 0 /simulator 0 /skip 0 /log 1
define logical simulator skip
let simulator pro%parse%simulator%narg.ge.0
let skip pro%parse%skip%narg.ge.0
define char*256 logfile
if pro%parse%log%narg.eq.1 then
  let logfile 'pro%parse%log%arg[1][6:]'  ! Strip off "file:"
endif
!
@ pipeline-tools
if simulator then
  @ pipeline-odp-init-simulator 'logfile'
else
  @ pipeline-odp-init-online 'logfile'
endif
if skip then
  @ pipeline-odp-process /skip ! Skip data already present
else
  @ pipeline-odp-process       ! Process data already present
endif
@ pipeline-odp-watch    ! Wait for new data and process it
