#!/proj/axaf/bin/perl -w

##########################################################################################
#
# Gather fid statistics
#
##########################################################################################

use File::Basename;
use Getopt::Long;
use RDB;
use lib '/proj/axaf/simul/lib/perl';
use PDL;
use Expect::Simple;
use CXC::Envs;
use Chandra::Tools::Common;
# use GrabEnv qw( grabenv diffenv envstr );
use TomUtil;

# Set up some constants

$HOME         = $ENV{PWD};
$tmp_stats    = 'TMP_stats.rdb';

@fidprops_cols = qw(slot mag_i_avg id_string id_num id_status);
%formats = (detector => '%6s',
	    ang_y_med => '%8.2f',
	    ang_z_med => '%8.2f',
	    ang_y_95th => '%8.2f',
	    ang_z_95th => '%8.2f',
	    ang_y_5th => '%8.2f',
	    ang_z_5th => '%8.2f',
	    mag_med => '%5.3f',
	    mag_i_avg => '%5.3f',
	    exp_time => '%7.2f',
	    sim_z_offset => '%6.1f',
	    tstart => '%12.1f',
	    );

##****************************************************************************
# MAIN PROGRAM
##****************************************************************************

get_options();			# Parse command line options
@obsid = get_obsids();		# Extract the list of obsids
set_env_variables();		# Set ascds vars

# Back up statistics file in case of problems
system("cp ${stats_db}.bak1 ${stats_db}.bak2") if (-e "${stats_db}.bak1");
system("cp ${stats_db} ${stats_db}.bak1") if (-e "${stats_db}");

foreach $obsid (@obsid) {
    $date = localtime;
    print "********* PROCESSING OBSID $obsid at $date ***********\n" if ($par{loud});
    init_stats_db();		# Initialize a temp celestial location database

    # Do some processing for each obsid.  If any step along the way fails, 
    # bail out to the clean-up step

    unless (get_fidprops_acacen_files()) {
	log_message($obsid, "Could not find necessary fidprops or acacen files");
	goto CLEAN_UP;
    }

    unless (merge_files()) {
	log_message($obsid, "Could not merge acacen files");
	goto CLEAN_UP;
    }

    unless (($fidpr_hdr, @fidprops) = read_fidprops()) {
	die;
    }

    unless (calc_stats()) {
	log_message($obsid, "Could not calculate statistics files");
	goto CLEAN_UP;
    }


  CLEAN_UP:
    close_stats_db();		# Copy the new entry into the cel loc DB and close
				# This minimizes data loss in case of processing problem
    clean_up() if ($par{clean});
}

chdir $HOME;

##****************************************************************************
sub get_options {
##****************************************************************************
    $par{database} = "$HOME/STATS.rdb";
    $par{loud}     = 1;		# Run loudly
    $par{plot}     = 1;		# Make plots
    $par{update}   = 1;		# Update the master table
    $par{dirs}     = $HOME;
    $par{clean}    = 1;

    GetOptions( \%par, 
	       'help!',
	       'loud!',
	       'dirs=s',
	       'tstart=s',
	       'tstop=s',
	       'database=s',
	       'dirs=s',
	       'update!',
	       'plot!',
	       'clean!',
	       ) ||
		   exit( 1 );

    if ($par{loud}) {
	print "COMMAND LINE PARAMETERS\n";
	foreach (sort keys %par) {
	    printf "  %-16s = %s\n", $_, $par{$_};
	}
	print "\n";
    }

    $stats_db = $par{database};

}

##****************************************************************************
sub get_obsids {
##****************************************************************************
    if (@ARGV && $ARGV[0] =~ /^@/) {
	$ARGV[0] =~ s/@//;
	@ARGV = grep !/^#/, `cat $ARGV[0]`;
	map {chomp; ($_) = split} @ARGV;
    }
    map {s/.*obs//; s/\///} @ARGV;

    if (exists $par{tstart}) {
	$par{tstart} = time2date(time+$par{tstart}*86400,1) if ($par{tstart} =~ /^-/);
	if ($par{tstop}) {
	    $par{tstop} = time2date(time+$par{tstop}*86400,1) if ($par{tstop} =~ /^-/);
	} else {
	    $par{tstop} = time2date(time,1);
	}

	print "Processing obsids from $par{tstart} to $par{tstop}\n\n" if ($par{loud});

	@obsfiles =  get_archive_files(tstart    => $par{tstart},
				       tstop     => $par{tstop},
				       prod      => "obspar",
				       file_glob => "axaf*obs0a.par*",
				       dir       => $HOME,
				       loud      => 0);
	unlink @obsfiles;
	push @ARGV, grep {$_ = $1+0 if (/axaff(\d+)/ && $1 < 60000)} @obsfiles;
	print "Will process obsids @ARGV\n";
    }

    return @ARGV;
}

##****************************************************************************
sub set_env_variables {
##****************************************************************************
#   %ciaox = grabenv('tcsh','source /soft/ciao/bin/ciao.csh');
#    %ciaox = grabenv('tcsh','source /proj/ascr4/DAtest/preInstalled/CXCDS_ROOT_SOLARIS/bin/ciao.csh -o');
#    $ciaox{ENVIRONMENT_NAME} = 'ciaox';
#    %ENV = %ciaox;			# For most commands use CIAOX
    %ENV = CXC::Envs::ftools();
}

##****************************************************************************
sub init_stats_db {
##****************************************************************************
# Read the existing output celestial location database header
    unless ($out_rdb_old) {
	$out_rdb_old = new RDB "${stats_db}" or die "Couldn't open ${stats_db}\n";
    }

# Open new one 

    $out_rdb = new RDB;
    $out_rdb->open("${tmp_stats}", ">") or die "Couldn't open $tmp_stats\n";
    $out_rdb->init($out_rdb_old);
}

##****************************************************************************
sub close_stats_db {
##****************************************************************************
    undef $out_rdb;		# Force RDB to tidy up file

    $sort_cols = "id_num date_obs obsid";
    $cmd = "sorttbl < $tmp_stats $sort_cols > TMP_XXX_QQQ.rdb";
    print "$cmd\n" if ($par{loud});
    system($cmd);
    system("mv TMP_XXX_QQQ.rdb $tmp_stats");

    # merge new info into master table, if desired
    if ($par{update}) {
	$cmd = "mergetbl < $stats_db $sort_cols $tmp_stats > TMP_XXX_QQQ.rdb"; 
	print "$cmd\n" if ($par{loud});
	system($cmd);
	system("mv TMP_XXX_QQQ.rdb $stats_db");
    }
}


##****************************************************************************
sub get_fidprops_acacen_files {
##****************************************************************************

# Get event file and source, either from current directory or from archive

    @acen = get_archive_files(obsid 	 => $obsid,
			      prod  	 => "asp1[*acen*]",
			      file_glob  => "pcad*acen1.fits*",
			      version    => [4,3,2,1]);
    return 0 unless (@acen);

    @fidpr = get_archive_files(obsid 	 => $obsid,
				prod  	 => "asp1[*fidpr*]",
			      file_glob => "pcad*fidpr1.fits*",
			      version    => [4,3,2,1]);
    return 0 unless (@fidpr);
}

##***************************************************************************
sub merge_files {
##***************************************************************************
# If necessary, merge centroid files
    $" = "\n";			# Set the list separator to "\n"
    foreach (qw(acen fidpr)) {
	if (@{$_} == 1) {
	    $cat = ($ {$_}[0] =~ /\.gz$/) ? "gunzip --stdout" : "cat"; 
	    $cmd = "$cat $ {$_}[0] > obs_${_}1.fits"; 
	    print "$cmd\n" if ($par{loud});
	    system($cmd);
        } else {
	    open (MERGE, "> merge.lis") || die "ERROR - Couldn't open merge.lis for writing\n";
	    print MERGE "@{$_}\n";
	    close MERGE;
	    $cmd = "fmerge \@merge.lis obs_${_}1.fits - clobber=yes";
	    print "$cmd\n" if ($par{loud});
	    system($cmd);
	    unlink "merge.lis";
	}
    }
    $" = ' ';			# Reset the list separator to " "
    return (-e "obs_acen1.fits" && -e "obs_fidpr1.fits");
}

##***************************************************************************
sub read_fidprops {
##***************************************************************************
    my @entries;
    my $i;
    my $j;
    my @vals;
    my %tmp_vals;
    my $fidpr_hdr;

    $fidpr_hdr = cfitsio_read_header("obs_fidpr1.fits","FIDPROPS");
    @vals = read_bintbl_cols("obs_fidpr1.fits", @fidprops_cols, { extname => 'FIDPROPS' });
    
    # Convert PDL elements into an array of hashes, for easier manipulation

    @entries = ();
    for $j (0 .. nelem($vals[0])-1) {
	%tmp_vals = ();
	for $i (0 .. $#fidprops_cols) {
	    $tmp_vals{$fidprops_cols[$i]} = (UNIVERSAL::isa($vals[$i],'PDL')) ?
		$vals[$i]->at($j) : $vals[$i]->[$j];
	}
	push @entries, { %tmp_vals };
    }
    return ($fidpr_hdr, @entries);
}

##***************************************************************************
sub clean_up {
##***************************************************************************
    my @files;

    @files = glob("pcad*fidpr*.fits* pcad*acen*fits* obs_fidpr1.fits obs_acen1.fits");
    unlink @files if (@files);
}

##***************************************************************************
sub calc_stats {
##***************************************************************************
    my %data = ();

    %fid_slots = ();
    foreach $fidpr (@fidprops) {
	$fid_slots{$fidpr->{slot}} = 1; # if ($fidpr->{id_status} eq 'GOOD');
	print "WARNING: Obsid $obsid has bad fid in slot $fidpr->{slot}\n" if ($fidpr->{id_status} ne 'GOOD');
    }
    @fid_slots = keys %fid_slots;
    return unless (@fid_slots);

    # Start up IDL if necessary
    print "Running IDL fid_stats procedure\n" if ($par{loud});
    unless ($idl) {
	$idl = new Expect::Simple { Cmd => "idl",
				    Prompt => 'IDL> ',
				    DisconnectCmd => 'exit',
				    Verbose => 0,
				    Debug => 0,
				    Timeout => 2000
				    };
    }

    # Read the centroid data and run the analysis script
    $idl->send('cen = mrdfits("obs_acen1.fits",1,hdr)');
    $idl->send("slots = [" . join(',',@fid_slots) . "]");
    $idl->send(".run fid_stats");

    $results = $idl->before();	# Results of script
    @results = split "\n", $results;

    # Now parse the results and put into output data hash
    foreach (@results) {
	# if an end of record is indicated, then output the data record
	if (/END_RECORD/) {
	    # Make data a bit prettier
	    $data{sim_z_offset} = ($fidpr_hdr->{LSI0STT3} + $fidpr_hdr->{STT0STF3}) * 20.493; # arcsec/mm
	    foreach (keys %data) {
		$data{$_} = sprintf $formats{$_} || "%s", $data{$_};
	    }
	    $out_rdb->write(\%data);
	    %data = ();
	}
	# Skip anything that is not a 'keyword = value' pair
	next unless /(\S+)\s*=\s*(\S+)/;
	($keyword, $value) = ($1, $2);
	$data{$keyword} = $value;

	# if the keyword is the slot, find the corresponding fidprops
	# entry and copy all the parameters to the output data hash
	if ($keyword eq 'slot') {
	    foreach $fidpr (@fidprops) {
		if ($fidpr->{slot} == $value) {
		    foreach (keys %{$fidpr}) {
			$data{$_} = $fidpr->{$_};
		    }
		    last;
		}
	    }
	}
    }
    return 1;
}

##***************************************************************************
sub log_message {
##***************************************************************************
    my ($obsid, $msg) = @_;

    open LOG, ">> $par{dirs}/LOG" or die "Couldn't open log file $par{dirs}/LOG\n";
    $date = localtime;
    printf LOG "$date: OBSID %5d: $msg\n", $obsid;
    printf("$date: OBSID %5d: $msg\n", $obsid) if ($par{loud});
    close LOG;
}