! Support for all-widget.ima, all-process.ima and all-self.ima
!
! Create the "all" structure and the processing scripts
!
define structure all /global
!
! Control parameters
define char*12 all%mode /global
define double all%range[3] all%minfres all%minband all%itime all%index /global
define integer all%phase_niter /like self_niter /global
define integer all%phase_times /like self_times /global
define integer all%ampli_niter[2] all%ampli_times[2] /global
define real all%collect /global
define integer all%drop[2] /global
define character*64 all%filter /global
define logical all%restore all%combine  all%resample /global
define character*12 all%telescope /global
define character*60 all%catalog /global
define character*8 all%next all%last /global
!
define logical all%doself /global
!
! Directories
define character*16 all%uvfits all%maps all%tables all%self all%raw /global
define character*16 all%prefix_self /global
define character*512 all%topdir /global
!
if file("all-memory.ima") then
  say 'ansi%red'"I-ALL,  Recovering parameters from all-memory.ima"'ansi%none'
  @ all-memory
else
  let all%mode SPLIT  ! Choice is CONTINUUM, WHOLE, SPLIT
  let all%phase_niter self_niter
  let all%phase_times self_times
  let all%ampli_niter 0 0
  let all%ampli_times 120 120
  let all%minfres 1   ! MHz
  let all%minband 250 ! MHz
  let all%filter "*"
  let all%restore yes ! Use UV_RESTORE
  let all%combine no
  let all%collect 0   ! Collection Flagging Threshold
  let all%drop 5      ! Drop 5 channels at each edge
  !
  ! Standard working directories
  let all%uvfits "UVFITS/"
  let all%self "SELF/"
  let all%maps "MAPS/"
  let all%tables "TABLES/"
  let all%raw "RAW/"
  !
  ! Default prefix for self-calibrated data
  let all%prefix_self "s-"
  let all%next "NONE"
  let all%last "NONE"
  let all%doself YES
endif
!
! Create the "RunTime" estimator structure
if exist(runtime) delete /var runtime
define structure runtime /global
define double runtime%start runtime%cur runtime%left /global
define char*16 runtime%end /global
begin procedure say-hms
  define char*4 unit
  define char*8 value
  if (&1.gt.90) then
    let &1 &1/60
    if (&1.gt.90) then
      let &1 &1/60
      let unit " hr_"
    else
      let unit " min"
    endif
  else
    let unit " sec"
  endif
  let value &1 /forma f8.1
  let &2 'value''unit'
end procedure say-hms
!
! Create the Scripts too
begin procedure do-time
  ! @ do-time TableUV InputDir OutputDir
  !
  !   Time averaging from directory &2 (RAW/ or SELF/)  
  !   to directory &3 (SELF/)
  read uv &2&1
  if (all%itime.le.0) then
    uv_time
  else
    uv_time min(180,all%itime)
  endif 
  write uv &3&1
end procedure do-time
!
begin procedure do-image
  ! @ do-image TableUV [Directory]
  !
  !   Imaging with separation of Spectral Lines and continuum 
  ! The behaviour depends on the existence of a spectral line
  ! catalog.
  if pro%narg.ge.2 then
    define character*256 cdir
    let cdir 'pro%arg[2]'"/"
  else
    define character*4 cdir
    let cdir "./"
  endif  
  !
  ! Image all spectral line windows  with all combinations of rest frequencies
  !
  define logical do_line
  define header a &1 read
  define character*6 itype
  !
  ! Produce spectra only if the spectral resolution is sufficient
  do_line = (abs(a%freqres).lt.all%minfres).and.(all%mode.ne."CONTINUUM")
  !
  define character in_file*256 in_ext*12 ou_file*256 in_dir*256
  define character post*2 ! "+D" for Detected, "+U" for Undetected, +C for Continuum
  !
  sic parse &1 in_file in_ext in_dir
  read uv 'in_dir''in_file'  /range all%drop[1] -all%drop[2] channel
  if exist(fields) then
    let itype "sky"
  else
    let itype "clean"
  endif
  uv_check beam   ! Required to drop edge channels
  if (all%mode.eq."CONTINUUM") then
    clear
    set box 4 28 5.5 17.5
  else if (all%mode.eq."SURVEY") then
    uv_preview ONLY
  else
    uv_preview
  endif
  set expand 1.0
  pen 1
  draw text -5 -3 "Imaging "'in_dir''in_file' 6 0 /char 1
  if (runtime%left.ne.0) then
    draw text 0 -3 "Time left "'runtime%end'"  &3" 4 0 /char 3
  else
    draw text 0 -3 "&3" 4 0 /char 3
  endif
  pen 0
  set expand 1.0
  !
  if (all%mode.ne."CONTINUUM").and.(exist(linedb%dbin)).and.(all%mode.ne."SURVEY") then
    !
    ! There is a Catalog - Only image identified spectral lines
    if preview%nfound.gt.0 then
      uv_filter /channel ! Filter the spectral lines (if any channel)
    endif
    !
    ! Save the continuum
    uv_map /cont /index 'all%index'
    clean 
    write cct 'cdir''in_file'+C
    if (all%restore) uv_restore
    write 'itype' 'cdir''in_file'+C
    uv_filter /reset 
    !
    if (preview%nfound.eq.0) return
    if (.not.do_line) return
    !
    define real f_scale
    if exist(redshift) then
      f_scale = 1+redshift
    else
      f_scale = 1
    endif
    !
    ! Save the continuum-free spectral lines if requested and possible.
    if (all%mode.eq."SPLIT") then 
      uv_baseline /channel ! Remove the continuum
    endif
    !
    for j 1 to preview%nfound
      specify frequency 'preview%found%freq[j]/f_scale' linename 'preview%found%lines[j]'
      on error next ! In case of Out of Range
      @ do-resample
      uv_check beam
      uv_map /range 'all%range[1]' 'all%range[2]' velo
      if (all%mode.eq."SPLIT") then
        post = "+U"
        clean
        if preview%ndetect.gt.0 then
          for k 1 to preview%ndetect
            if preview%detected%freq[k].eq.preview%found%freq[j] then
              post = "+D"
              break
            endif
          next
        endif
      else
        post = "+A"
        clean /restart 'cdir''in_file'+C.cct
      endif
      write cct 'cdir''in_file'-'preview%found%species[j]'"_"'j''post'
      if (all%restore) uv_restore
      write 'itype' 'cdir''in_file'-'preview%found%species[j]'"_"'j''post'
      !
      uv_resample ! Get back to original UV data
    next
    on error
    !
  else
    !
    ! TSurvey mode or No Catalog - image the whole band
    if exist(preview%nchannels) then
      uv_filter /channel ! Filter the spectral lines (if any channel)
    endif
    !
    ! Save the continuum
    uv_map /cont /index 'all%index'
    clean 
    write cct 'cdir''in_file'+C
    if (all%restore) uv_restore
    write 'itype' 'cdir''in_file'+C
    uv_filter /reset
    !
    if (.not.do_line) return
    ! Save the continuum-free spectral lines if requested and possible.
    if (all%mode.eq."SPLIT") 
      uv_baseline /channel ! Remove the continuum
    endif
    specify frequency preview%freq  linename 'nint(preview%freq)'
    @ do-resample
    uv_map
    if (all%mode.eq."SPLIT") then
      clean
    else
      clean /restart 'cdir''in_file'+C.cct
    endif
    if (all%restore) uv_restore
    !
    if (all%mode.eq."SPLIT") then
      write cct 'cdir''in_file'+U
      write 'itype' 'cdir''in_file'+U    
      ! Add back the Continuum
      map_combine 'cdir''in_file'+A.lmv-clean ADD CLEAN CONTINUUM 
    else
      write cct 'cdir''in_file'+A
      write 'itype' 'cdir''in_file'+A    
    endif
  endif
end procedure do-image
!
begin procedure do-extract
  ! @ do-extract TableUV [Directory]
  !
  ! Table extraction with separation of Spectral Lines and
  ! continuum Tables according to all%mode
  if pro%narg.eq.2 then
    define character*256 cdir
    let cdir 'pro%arg[2]'"/"
  else
    define character*4 cdir
    let cdir "./"
  endif  
  !
  ! Extract one spectral line UV table per spectral line, with
  ! the appropriate spectral range
  ! 
  define logical do_line
  define header a &1 read
  do_line = (abs(a%freqres).lt.all%minfres).and.all%mode.ne."CONTINUUM"
  define real sample[3]
  !
  define character in_file*256 in_ext*12 ou_file*256 in_dir*256
  define character post*2
  !
  sic parse &1 in_file in_ext in_dir
  !
  read uv 'in_dir''in_file' /range all%drop[1] -all%drop[2] channel
  if (all%mode.ne."SURVEY") then
    uv_preview
  else
    uv_preview ONLY
  endif
  set expand 1.4
  draw text 0 -3 "Extracting "'in_dir''in_file' 5 0 /char 2
  set expand 1.0
  !
  if (all%mode.ne."CONTINUUM").and.(exist(linedb%dbin)) then
    if (preview%nfound.gt.0).and.do_line then
      define real f_scale
      if exist(redshift) then
        f_scale = 1+redshift
      else
        f_scale = 1
      endif
      if (all%mode.eq."SPLIT") uv_baseline  ! Remove continuum
      !
      for j 1 to preview%nfound
        if (all%mode.eq."SPLIT") then
          post = "+U"
          if preview%ndetect.gt.0 then
            for k 1 to preview%ndetect
              if preview%detected%freq[k].eq.preview%found%freq[j] then
                post = "+D"
                break
              endif
            next
          endif
        else
          post = "+A"
        endif
        specify linename 'preview%found%lines[j]' frequency 'preview%found%freq[j]/f_scale'
        on error next ! In case of Out of Range
        if (all%resample) then
          @ do-resample
          write uv 'cdir''in_file'-'preview%found%species[j]'"_"'j''post' 
          uv_resample ! Get back to original, baseline subtracted
        else
          write uv 'cdir''in_file'-'preview%found%species[j]'"_"'j''post' /range 'all%range[1]' 'all%range[2]' velo
        endif
        on error 
      next
      !
      ! Reload the UV data
      if (all%mode.eq."SPLIT") read uv 'in_dir''in_file' /range all%drop[1] -all%drop[2] channel 
    endif  
  else if (do_line) then
    !
    ! No catalog - Create spectral line UV table if needed
    ! Save the continuum-free spectral lines if requested and possible.
    if (all%mode.eq."SPLIT") then
      uv_baseline /channel ! Remove the continuum
      post = "+L"
    else
      post = "+A"
    endif
    let sample all%range
    let all%range[1:2] 0
    @ do-resample
    let all%range sample
    write uv 'cdir''in_file''post'
  endif
  !
  if (all%mode.ne."CONTINUUM") then
    uv_filter /channel        ! Remove line
  endif
  uv_compress /continuum      ! Compute the associated continuum
  write uv 'cdir''in_file'"+C.uvt"   ! Save the associated continuum
end procedure do-extract
!
begin procedure do-resample
  ! Return if no Velocity Resolution specified, or if interpolation would be needed
  if all(all%range.eq.0) return
  !
  define real vres
  let vres max(abs(uv%velres),all%range[3])
  !
  ! Resample at the desired spectral resolution
  define integer nc
  let nc nint(all%range[2]-all%range[1])/vres 
  !
  if (nc.eq.0).or.(.not.exist(linedb%dbin)) then
    ! Check if any resampling still needed
    if (vres.eq.abs(uv%velres)) return
    ! Whole covered band at specified resolution
    uv_resample * * * vres 
  else
    ! Specified velocity range at specified resolution
    uv_resample NC (nc+1.)/2. (all%range[1]+all%range[2])/2 vres 
  endif
end procedure do-resample
!
begin procedure do-sky
  ! @ do-sky Name [Directory]
  !
  !   Apply Primary beam correction
  define character in_file*256 in_ext*12 ou_file*512 in_dir*256
  !
  sic parse &2&1 in_file in_ext in_dir
  !
  let ou_file 'in_dir''in_file'".lmv-clean"
  if file(ou_file) then
    read clean 'in_dir''in_file'
    primary
    write sky 'in_dir''in_file'
  endif
end procedure do-sky
!
!----------------------------------------------------------------------------
!
! Self-calibration scripts
!
begin procedure sub-self-check
  ! @ sub-self-check TableUV
  !
  !   Check if Self-calibration is possible on a UV Table
  !
  read uv &1        ! Read UV data
  if (exist(fields)) then
    message w SELF "&1 is a mosaic, no Self-calibration possible"
    let all%doself NO
    return
  endif
  uv_check          ! Verify edges
  if (all%mode.ne."CONTINUUM").and.(all%mode.ne."SURVEY") then
    uv_preview        ! Search for spectral lines
    uv_filter /cha    ! Keep only continuum
  endif
  uv_map /cont
  !
  define real st sn
  if (self_threshold.le.0) then
    let st -1./beam%min
  else
    let st self_threshold
  endif
  !
  ! If S/N is less < ST, assume you cannot self-calibrate
  !
  let sn dirty%max/dirty%noise
  show dirty
  if sn.lt.st then
    message E SELFCAL "Signal to noise "'0.1*nint(10*sn)'" is below threshold "'0.1*nint(10*st)'" for Self-Calibration"
    let mapping_error YES
    let all%doself NO
  else
    message I SELFCAL "Expected S/N ratio "'nint(sn)'
  endif
  !
end procedure sub-self-check
!
begin procedure sub-self-comp
  if .not.all%doself then
    message I SELF "All%DoSelf is NO, skipping Self-calibration"
    return
  endif
  ! @ sub-self-comp TableUV
  !
  !   Self calibrate in PHASE and AMPLITUDE a UV Table
  !
  let all%combine no  ! If you re-compute, you must re-collect
  read uv &1        ! Read UV data
  if (exist(fields)) then
    message w SELF "&1 is a mosaic, no Self-calibration possible"
    return 
  endif
  uv_check          ! Verify edges
  if (all%mode.ne."CONTINUUM").and.(all%mode.ne."SURVEY") then
    uv_preview        ! Search for spectral lines
    uv_filter /cha    ! Keep only continuum
  endif
  uv_map /cont      ! Make a first image
  statistic dirty
  define real st sn
  if (self_threshold.le.0) then
    let st -1./beam%min
  else
    let st self_threshold
  endif
  let sn dirty%max/dirty%noise
  !
  ! If S/N is less < ST, assume you cannot self-calibrate
  !
  if (sn.lt.st) then
    message E SELFCAL "Signal to noise "'nint(sn)'" is insufficient (less than "'nint(st)'") "
    let mapping_error YES
    return 
  else
    clean
    show clean
    PAUSE
  endif
  !
  ! Perform Phase Self-Calibration
  !   Parameters should be adjusted by user
  let self_mode PHASE
  delet /var self_times self_niter
  define integer self_times self_niter /like all%phase_times /global
  let self_times all%phase_times 
  let self_niter all%phase_niter  
  selfcal ?
  PAUSE "Adjust self_niter and/or self_times if needed"
  selfcal PHASE
  PAUSE "Type CONTINUE to save and go on with Amplitude"
  sic copy 'self_sname'.gdf &1-antenna.phase
  selfcal SAVE
  sic copy selfcal-PHASE.last selfcal-PHASE-&1.last
  write cgains &1.phase
  !
  ! Now perform Amplitude Self-Calibration 
  selfcal apply
  delet /var self_times self_niter
  define integer self_times self_niter /like all%ampli_times /global
  let self_times all%ampli_times
  let self_niter all%ampli_niter
  selfcal AMPLI
  PAUSE "Type CONTINUE to save and finish file &1"
  sic copy 'self_sname'.gdf &1-antenna.ampli
  selfcal SAVE
  sic copy selfcal-AMPLI.last selfcal-AMPLI-&1.last
  write cgains &1.ampli
  !
end procedure sub-self-comp
!
begin procedure sub-self-show
  define char*128 self_remember
  let self_remember 'self_sname'
  !
  if file("&1-antenna.phase") then
    let self_sname &1-antenna.phase
    let self_mode PHASE
    selfcal SHOW 10
    if (self_hard) hardcopy &1-phase /dev epdf /over
  endif
  if file("&1-antenna.ampli") then
    let self_sname &1-antenna.ampli
    let self_mode AMPLI
    selfcal SHOW 10
    if (self_hard) hardcopy &1-ampli /dev epdf /over
  endif
  let self_sname 'self_remember'
end procedure sub-self-show
!
begin procedure sub-self-collect
  ! @ sub-self-collect 
  !
  !   Collect self calibration computed by  @ sub-self-comp
  !   for several wide bands as a delay and frequency independent
  !   amplitude correction
  !
  if .not.all%doself then
    message I SELF "All%DoSelf is NO, skipping Self-calibration"
    return
  endif
  define char*256 rawdir selfdir
  let all%topdir 'sic%directory'"/"  
  !
  on error "@ sub-self-return"
  !
  let all%combine no
  sic dir 'all%raw'
  if file("self-antenna.phase") sic delete self-antenna.phase
  sic find *-antenna.phase
  if (dir%nfile.eq.0) then
    message W COLLECT "No Phase self-calibration"
  else
    collect *-antenna.phase self-antenna.phase /flag 'all%collect'
    derive self-antenna.phase self-baseline.phase
  endif
  !
  if file("self-antenna.ampli") sic delete self-antenna.ampli
  sic find *-antenna.ampli
  if (dir%nfile.eq.0) then
    message W COLLECT "No Amplitude self-calibration"
  else
    collect *-antenna.ampli self-antenna.ampli /flag 'all%collect'
    derive self-antenna.ampli self-baseline.ampli
  endif
  !
  let all%combine yes
  sic dir 'all%topdir'
end procedure sub-self-collect
!
begin procedure sub-self-return
  sic dir 'all%topdir'
  return base
end procedure sub-self-return
!
begin procedure sub-self-apply
  ! @ sub-self-apply TableUV [Number]
  !
  !   Apply self calibration computed by  @ sub-self-comp
  !   or sub-self-collect (depending on all%combine)
  !   to a UV table and all UV tables falling in its
  !   frequency range
  !
  if .not.all%doself then
    message I SELF "All%DoSelf is NO, skipping Self-calibration"
    return
  endif
  !
  define char*6 chain
  define double freq
  define char*512 rawdir selfdir
  define char*32 fich
  define char*8 akey
  !
  let all%topdir 'sic%directory'"/"  
  sic mkdir 'all%self'
  sic dir 'all%self'
  let selfdir 'sic%directory'"/"
  !
  sic dir 'all%topdir'
  sic dir 'all%raw'
  !
  if (all%combine) then
    let fich self-baseline
    let akey DELAY
  else
    let fich &1
    let akey PHASE
  endif
  !
  ! Do nothing if no Phase solution
  define character*48 fich2
  let fich2 'fich'".phase"
  if .not.file(fich2) then
    message W APPLY "No such Phase self-calibration solution "'fich'".phase"
    sic dir 'all%topdir'
    return
  endif
  !
  read uv &1
  uv_preview ONLY ! Just to see things, no filtering required
  let freq (preview%fmin+preview%fmax)/2.0 
  let chain nint(freq) /format I6
  specify frequency freq line "C-"'chain'
  read cgains 'fich'.phase
  apply 'akey' /flag
  read cgains 'fich'.ampli
  apply ampli /flag
  write uv 'selfdir''all%prefix_self'&1
  !
  ! Update Frequency range in Combine mode to locate all
  ! narrow bands if not already done - This must be done
  ! only once, so the second argument is used as a counter
  ! to figure out if it has already been done.
  if (all%combine) then
    if ((&2+0).gt.1) then
      sic dir 'all%topdir'
      return
    endif
    @ sub-find-frequency *
  endif
  @ noema-scan 'all%filter' &3    
  !
  for i 1 to noema%ntables
    read uv 'noema%table[i]'
    specify frequency noema%freq[i] linename 'noema%line[i]'
    read cgains 'fich'.phase
    apply 'akey' /flag
    read cgains 'fich'.ampli
    apply ampli /flag
    write uv 'selfdir''all%prefix_self''noema%table[i]'
  next
  sic dir 'all%topdir'
end procedure sub-self-apply
!
!
begin procedure sub-find-frequency
  ! @ sub-find-frequency TableFilter
  ! 
  !   Find the overall Frequency range of all UV tables
  !
  define double fmin fmax fa fb finf fsup
  !
  sic find &1.uvt
  let fmin 1E36
  let fmax -1
  !
  for fich /in dir%file
    define header tmp 'fich' read 
    let fa (0.5-tmp%convert[1,1])*tmp%freqres+tmp%restfre
    let fb (tmp%nchan+0.5-tmp%convert[1,1])*tmp%freqres+tmp%restfre
    let finf min(fa,fb)
    let fsup max(fa,fb)
    !
    let fmin min(fmin,finf)
    let fmax max(fmax,fsup)
    delete /var tmp
  next
  !
  greg1\limits fmin fmax = = 
end procedure sub-find-frequency
