#! /bin/tcsh 
## make_corrected_bcds.tcsh
## Driver script to perform preprocessing steps on IRAC BCDs to mitigate 
## various data artifacts
##
## You will need to edit this script to get it to correct specific sets of 
## observations.  The default settings are for a non-HDR AOR without repeats
##
## Comments containing the word CHECK indicate that you should decide whether
## or not to run that module
##
## Comments containing the word HDR indicate that module should be run on HDR
## data only
##
## Comments containing the word REPEAT indicate that the module should be run
## only if your data taking uses repeats.  In addition, you will need to
## comment out the non-repeat versions of these modules
## 
## Comments containing the word FULL_FRAME indicate that the module should be
## run on full frame AORs (non-HDR) only
## 
## History:
##    More status messages added 27 July 2005 SJC
##    External log file produced 30 Aug 2005 SJC

## CHECK
## You will need to comment out the next 5 lines after making sure the rest of
## the script does what you want.
echo "************************************************"
echo " Check that the processing steps are as desired"
echo " Please edit this script file!!!!"
echo " If you have fullframe time data, this script will run as is, after"
echo " commenting out this initial message and the exit command following it"
echo " If you have HDR data, enable the HDR mode modules"
echo " If your data contains repeats in HDR mode, enable the modules handling"
echo " repeats"
echo "************************************************"
exit

## Directory containing artifact correction code -- At present, do not change
set pdir=./irac_preprocess

echo "Correcting BCDs start at : "`date`
echo "Correcting BCDs start at : "`date` >> make_corrected_bcds_$$.log

## CHECK
## Remove previous attempt at artifact correction. 
## This can be commented out, if you like as all of the FITS files will be
## overwritten with the new attempt in any case
echo "Cleaning up previous attempt : "`date`
echo "Cleaning up previous attempt : "`date` >> make_corrected_bcds_$$.log
${pdir}/clean_all.tcsh >& clean_all.log
echo "Finished cleaning up previous attempt : "`date`
echo "Finished cleaning up previous attempt : "`date` >> make_corrected_bcds_$$.log

## CHECK
## Shell script to copy original BCDs
## Script assumes that all data resides in subdirectories.  That is, you are
## running this script from the rREQKEY directory unzipped from the Leopard 
## download.
## This command can be commented out after the first time the script is run.
echo "Copying original files : "`date`
echo "Copying original files : "`date` >> make_corrected_bcds_$$.log
${pdir}/get_originals.tcsh >& get_originals.log
echo "Finished copying original files : "`date`
echo "Finished copying original files : "`date` >> make_corrected_bcds_$$.log

## Create files containing lists of images to process
echo "Making lists of BCD images, uncertainties and masks : "`date`
echo "Making lists of BCD images, uncertainties and masks : "`date` \
               >> make_corrected_bcds_$$.log
idl ${pdir}/make_processing_lists.idl >& make_processing_lists.log
echo "Finished with file list creation : "`date`
echo "Finished with file list creation : "`date`  >> make_corrected_bcds_$$.log
## CHECK
## You may want to edit the lists before proceeding, if there are images that
## you know are bad.

## Create lists containing data in all channels
echo "Generating 4 channel file lists : "`date`
echo "Generating 4 channel file lists : "`date`  >> make_corrected_bcds_$$.log
${pdir}/make_big_lists.tcsh
echo "Finished with 4 channel file lists : "`date`
echo "Finished with 4 channel file lists : "`date` >> make_corrected_bcds_$$.log

## Create imasks from the dmask files
echo "Create imasks : "`date`
echo "Create imasks : "`date`  >> make_corrected_bcds_$$.log
idl ${pdir}/make_imasks_from_dmasks.idl >& make_imasks_from_dmasks.log
echo "Finished creating imasks : "`date`
echo "Finished creating imasks : "`date` >> make_corrected_bcds_$$.log

## Copy BCDs and uncertainty images to sfx_ files, so that a version of the 
## original BCDs exist in the working directory
idl ${pdir}/make_sfx_versions.idl >& make_sfx_versions.log

## From this point onward, use the imasks in processing
echo "Update processing lists with imask filenames : "`date`
echo "Update processing lists with imask filenames : "`date`  \
           >> make_corrected_bcds_$$.log
idl ${pdir}/update_processing_lists.idl >& update_processing_lists.log
${pdir}/make_big_lists.tcsh
echo "Finished updating processing lists with imask filenames : "`date`
echo "Finished updating processing lists with imask filenames : "`date` \
           >> make_corrected_bcds_$$.log

## Add zody contribution of skydark back
echo "Add zody subtracted by sky dark back to bcds : "`date`
echo "Add zody subtracted by sky dark back to bcds : "`date` \
           >> make_corrected_bcds_$$.log
idl ${pdir}/add_darkback.idl >& add_darkback.log
echo "Finished with zody addition : "`date`
echo "Finished with zody addition : "`date`  >> make_corrected_bcds_$$.log

## add the pmask info to the imasks
echo "Adding pmask info to imasks : "`date`
echo "Adding pmask info to imasks : "`date`  >> make_corrected_bcds_$$.log
idl ${pdir}/add_pmasks_to_imasks.idl >& add_pmasks_to_imasks.log
echo "Finished adding pmask info to imasks : "`date`
echo "Finished adding pmask info to imasks : "`date`  \
           >> make_corrected_bcds_$$.log

## If HDR data, you'll need some additional processing, otherwise comment
## out the scripts pertaining to HDR data

## HDR only
## First match the the background levels between medium and long frames
#echo "Matching HDR medium frame median to long frame medians : "`date`
#echo "Matching HDR medium frame median to long frame medians : "`date` \
#          >> make_corrected_bcds_$$.log
#idl ${pdir}/match_hdr_frames_med.idl >& match_hdr_frames_med.log
#echo "Finished matching HDR medium frame median to long frame medians : "`date`
#echo "Finished matching HDR medium frame median to long frame medians : "\
#         `date` >> make_corrected_bcds_$$.log

## If your observation uses repeats, then you will have to match the medium
## frame to the first long frame in each repeat set.  This requires a little 
## different list management.

## REPEAT only HDR only
## First you will need lists containing just one repeat for each long frame
## and example of how to make these lists is
#${pdir}/extras/split_long_list2.tcsh >& split_long_list2.tcsh
## Now match the medium frame time to the first repeat of the long frame time
#echo "Matching HDR medium frame median to long frame medians : "`date`
#echo "Matching HDR medium frame median to long frame medians : "`date` \
#          >> make_corrected_bcds_$$.log
#idl ${pdir}/match_hdr_frames_med1.idl >& match_hdr_frames_med1.log
#echo "Finished matching HDR medium frame median to long frame medians : "`date`
#echo "Finished matching HDR medium frame median to long frame medians : "\
#         `date`  >> make_corrected_bcds_$$.log

## if there are short, medium and long frame times (100s HDR data), you will 
## need to run the script twice once to match medium to long and then to match 
## the short to long.
## 100s HDR only
#echo "Matching HDR short frame median to long frame medians : "`date`
#echo "Matching HDR short frame median to long frame medians : "`date` \
#           >> make_corrected_bcds_$$.log
#idl ${pdir}/match_hdr_frames_short.idl >& match_hdr_frames_short.log
#echo "Finished matching HDR short frame median to long frame medians : "`date`
#echo "Finished matching HDR short frame median to long frame medians : "\
#           >> make_corrected_bcds_$$.log`date`

## If your observation uses repeats, then you will have to match the short
## frame to the first long frame in each repeat set.  This requires a little 
## different list management.
## HDR only REPEATS only
## Now match the short frame time to the first repeat of the long frame time
#echo "Matching HDR medium frame median to long frame medians : "`date`
#echo "Matching HDR medium frame median to long frame medians : "`date` \
#           >> make_corrected_bcds_$$.log
#idl ${pdir}/match_hdr_frames_short1.idl >& match_hdr_frames_short1.log
#echo "Finished matching HDR medium frame median to long frame medians : "`date`
#echo "Finished matching HDR medium frame median to long frame medians : "\
#          `date`  >> make_corrected_bcds_$$.log

## If you have 100s HDR, you'll want to fix the saturations in the medium 
## data using the short data.
## 100s HDR only
#echo "Fixing HDR medium frame saturations : "`date`
#echo "Fixing HDR medium frame saturations : "`date`  \
#          >> make_corrected_bcds_$$.log
#idl ${pdir}/quickhdr_sat_fix_short.idl >& quickhdr_sat_fix_short.log
#echo "Finished fixing HDR medium frame saturations : "`date`
#echo "Finished fixing HDR medium frame saturations : "`date` \
#           >> make_corrected_bcds_$$.log

## Now fix saturations in HDR long frames using medium data
## HDR only
#echo "Fixing HDR long frame saturations : "`date`
#echo "Fixing HDR long frame saturations : "`date` \
#           >> make_corrected_bcds_$$.log
#idl ${pdir}/quickhdr_sat_fix.idl >& quickhdr_sat_fix.log
#echo "Finished fixing HDR long frame saturations : "`date`
#echo "Finished fixing HDR long frame saturations : "`date` \
#           >> make_corrected_bcds_$$.log

## If you are using repeats, then you need to correct each set of repeats
## separately -- note that you will have to make the appropriate filelists.
## for repeat 1
## HDR only REPEAT only
#echo "Fixing HDR long frame saturations : "`date`
#echo "Fixing HDR long frame saturations : "`date` \
#            >> make_corrected_bcds_$$.log
#idl ${pdir}/quickhdr_sat_fix1.idl >& quickhdr_sat_fix1.log
#echo "Finished fixing HDR long frame saturations : "`date`
#echo "Finished fixing HDR long frame saturations : "`date` \
#            >> make_corrected_bcds_$$.log
## for repeat 2
#echo "Fixing HDR long frame saturations : "`date`
#echo "Fixing HDR long frame saturations : "`date` \
#            >> make_corrected_bcds_$$.log
#idl ${pdir}/quickhdr_sat_fix2.idl >& quickhdr_sat_fix2.log
#echo "Finished fixing HDR long frame saturations : "`date`
#echo "Finished fixing HDR long frame saturations : "`date` \
#            >> make_corrected_bcds_$$.log
## do something similar for subsequent repeats


## Need another module to estimate saturations for non-HDR data
## Still working on this.  For now, super-saturated sources in 
## non-HDR data will not have the artifacts that they produce mitigated.
## NORMAL only 

## Update imasks with artifacts as determined from positions of triggering
## pixels
echo "Updating imasks with artifacts : "`date`
echo "Updating imasks with artifacts : "`date`  >> make_corrected_bcds_$$.log
idl ${pdir}/make_artifact_masks.idl >& make_artifact_masks.log
echo "Finished updating imasks with artifacts : "`date`
echo "Finished updating imasks with artifacts : "`date` \
               >> make_corrected_bcds_$$.log

## Create images with estimate of "true" sky in masked pixels
echo "Making estimated sky images : "`date`  >> make_corrected_bcds_$$.log
echo "Making estimated sky images : "`date`
idl ${pdir}/make_estimated_images.idl >& make_estimated_images.log
echo "Finished making estimated sky images : "`date`
echo "Finished making estimated sky images : "`date` ## do something similar
for subsequent repeats


## Need another module to estimate saturations for non-HDR data
## Still working on this.  For now, super-saturated sources in
## non-HDR data will not have the artifacts that they produce mitigated.
## NORMAL only

## Update imasks with artifacts as determined from positions of triggering
## pixels
echo "Updating imasks with artifacts : "`date`
echo "Updating imasks with artifacts : "`date`  >> make_corrected_bcds_$$.log
idl ${pdir}/make_artifact_masks.idl >& make_artifact_masks.log
echo "Finished updating imasks with artifacts : "`date`
echo "Finished updating imasks with artifacts : "`date` \
               >> make_corrected_bcds_$$.log

## CHECK
## Either create a model from your data or use a supplied one, 
## If you do create a model, uncomment the next three lines and edit
## make_muxmodel_driver.idl so that the filenames are to your liking
## Please check the resulting models to make sure they make sense.
#echo "Creating muxbleed models : "`date`
#echo "Creating muxbleed models : "`date` >> make_corrected_bcds_$$.log
#idl ${pdir}/make_muxmodel_driver.idl >& make_muxmodel_driver.log
#echo "Finished with muxbleed models : "`date`
#echo "Finished with muxbleed models : "`date` >> make_corrected_bcds_$$.log

## CHECK
## The muxbleed models returned in the previous step may be noisy and/or
## have bad values.  Please view the model files and correct if necessary
## Sample scripts to fix up muxbleed models

#echo "Editing channel 1 muxbleed model : "`date`
#echo "Editing channel 1 muxbleed model : "`date` \
#               >> make_corrected_bcds_$$.log
#idl ${pdir}/adjust_muxmodel_ch1.idl >& adjust_muxmodel_ch1.log
#echo "Finished editing channel 1 muxbleed model : "`date`
#echo "Finished editing channel 1 muxbleed model : "`date` \
#               >> make_corrected_bcds_$$.log

#echo "Editing channel 2 muxbleed model : "`date`
#echo "Editing channel 2 muxbleed model : "`date` \
#               >> make_corrected_bcds_$$.log
#idl ${pdir}/adjust_muxmodel_ch2.idl >& adjust_muxmodel_ch2.log
#echo "Finished editing channel 2 muxbleed model : "`date`
#echo "Finished editing channel 2 muxbleed model : "`date` \
#               >> make_corrected_bcds_$$.log

## Correct the artifacts
echo "Performing artifact correction : "`date`
echo "Performing artifact correction : "`date` >> make_corrected_bcds_$$.log
idl ${pdir}/do_fit_artifacts.idl >& do_fit_artifacts.log
echo "Finished with artifact correction : "`date`
echo "Finished with artifact correction : "`date` >> make_corrected_bcds_$$.log

## Update the lists to use the corrected images
echo "Updating processing lists to use corrected images : "`date`
echo "Updating processing lists to use corrected images : "`date` \
                >> make_corrected_bcds_$$.log
idl ${pdir}/cor_update_processing_lists.idl >& cor_update_processing_lists.log
${pdir}/make_big_lists.tcsh
echo "Finished updating processing lists to use corrected images : "`date`
echo "Finished updating processing lists to use corrected images : "`date` \
                >> make_corrected_bcds_$$.log
echo "Please check the corrected images!!!"
echo "Please check the corrected images!!!" >> make_corrected_bcds_$$.log


## CHECK
## Now perform overlap correction using the IDL scripts
## The SSC post-BCD tool overlap.pl can also be used.  The IDL procedure may be
## slightly more robust.  If you are not satisfied with the results of the IDL
## routine, please try overlap.pl
echo "Performing overlap correction : "`date`
echo "Performing overlap correction : "`date` >> make_corrected_bcds_$$.log
idl ${pdir}/do_correct_overlap.idl >& do_correct_overlap.log
echo "Finished with overlap correction : "`date`
echo "Finished with overlap correction : "`date` >> make_corrected_bcds_$$.log

## Update the lists to use the overlap corrected images
echo "Updating processing lists to use overlap corrected images : "`date`
echo "Updating processing lists to use overlap corrected images : "`date` \
                  >> make_corrected_bcds_$$.log
idl ${pdir}/over_update_processing_lists.idl >& over_update_processing_lists.log
${pdir}/make_big_lists.tcsh
echo "Finished updating processing lists to use overlap corrected images : "`date`
echo "Finished updating processing lists to use overlap corrected images : "\
                  `date`  >> make_corrected_bcds_$$.log

echo "Finished with correcting BCDs at : "`date`
echo "Finished with correcting BCDs at : "`date` >> make_corrected_bcds_$$.log
echo "Time to get started with mosaicing and source extraction!"
echo "Time to get started with mosaicing and source extraction!" \
                  >> make_corrected_bcds_$$.log
echo "Good Luck!!" 
echo "Good Luck!!"  >> make_corrected_bcds_$$.log
