#!/usr/bin/perl -w
# (c) 2012/2013 Joerg Mann / 1&1 Internet AG
#
# last update at now ...

###
use warnings;
use strict;
use English;
use Getopt::Long;
use Term::ANSIColor;
use Date::Language;
use POSIX qw(strftime);
use File::Basename;
binmode STDOUT, ":utf8";

### defaults
my $version        = "0.073";
my $alife_timeout  = "30";	# sec for remote-nodes timeout
my $is_tty 	   = 0;
my $mars_dir       = '/mars';
my $himself        = `uname -n` or die "cannot determine my network node name\n";
my $clearscreen	   = `clear`;
my $MarsTreeVer    = 0.1;
chomp $himself;

### ARGV
# Optionen in Hash-Ref parsen
my $params = {};
GetOptions( $params, 'help', 'h', 'version', 'v', 'resource=s', 'system', 'interval=i', 'history', 'ascii', 'debug' );

#########################################################################################
### terminal settings
my $Color_blue     = 'yellow';
my $Color_green	   = 'green';
my $Color_red	   = 'red';
my $Gls = "\x{2551}";
my $Glw = "\x{2550}";
my $Gkr = "\x{2560}";
my $Gao = "\x{255A}";
my $Gau = "\x{2554}";
if ( $params->{ascii} ) {
    $Gls = "|";
    $Glw = "-";
    $Gkr = "+";
    $Gao = "+";
    $Gau = "+";
}
my $Gab = "$Glw$Glw$Glw> ";
my $Gfr = "       ";


#########################################################################################
### small help
sub display_help {
    my $HelpText = shift;
    print "$HelpText\n\n" if ($HelpText);
    print "Usage: mars-status [--help]\n";
    print "Usage: mars-status [--version]\n";
    print "Usage: mars-status (without specification of parameters, an abstract of all the information sent)\n";
    print "Usage: mars-status [--resource <RESNAME>] [--interval <seconds>] | [--history] | [--debug ] | [--system]\n";
    print "       --resource : limits the display to the specified resource\n";
    print "       --interval : refreshes the display every second xxx\n";
    print "       --history  : shows information about the log files, version numbers and their status\n";
    print "       --ascii    : display history in ascii code letters\n";
    print "       --system   : display mars-system informations\n";
    print "       --debug    : additional display debug messages\n\n";
    print "Usage small include refresh : mars-status --interval 2\n";
    print "Usage full, include debug   : mars-status --system --history --debug\n\n";
    exit;
}

if($params->{help} || $params->{h} ) {
    display_help;
}

# Farbe zuruecksetzen
$SIG{INT} = sub {
    print color 'reset';
    print $clearscreen;
    exit;	
};

#########################################################################################
### figure out TTY
my $tty = readlink '/dev/stdout';
while ( my $temp = readlink $tty ) {
    $tty = $temp;
}
if ( $tty =~ /^\/dev\/pts\// ) {
    $is_tty = 1;
} elsif ( $tty =~ /^\/dev\/tty/ ) {
    $is_tty = 1;
}



#########################################################################################
### print color
sub print_screen {
    my $Text  = shift;
    my $Color = shift;
    my $Level = shift;

    $Color = 'FAINT' if (!$Color);
    print color "$Color" if ( $is_tty );
    print "$Text";
    print color 'reset' if ( $is_tty );
}


#########################################################################################
### read link
sub check_link {
    my $dir = shift;
    my $result = readlink $dir;
    if ( !$result ) {
	return 0;
    } else {
	return $result;
    }
}


#########################################################################################
### read links
sub convert_link {
    my $link = shift;
    $link    = check_link "$link";
    if (( !$link ) || ( $link eq 0 )) {
        print_screen "off", "$Color_red";
    } else {
	print_screen "on", "$Color_green";
    }
    return $link;
}


#########################################################################################
### sub display resource-partner
sub display_partner {
    my %p 		= @_;
    my $PRes  		= $p{ressource};
    my $PName 		= $p{nodename};
    my $PSize 		= $p{ressource_size};
    my $ref_ResPartner  = $p{res_partner};
    my $ref_AULogfile	= $p{res_AULogfile};
    my $PStatus  	= check_link "$mars_dir/$PRes/primary";
    my $PDevice  	= check_link "$mars_dir/$PRes/device-$PName";  
    my $Ljoined  	= check_link "$mars_dir/$PRes/device-$himself";
    
    ##########################################################################
    ### check status
    if ( $PStatus eq $PName ) {   
        print_screen "Primary", "$Color_blue bold";
    } else {
        if ( $PDevice eq 0 ) {
            print_screen "not joined","$Color_red";
            print_screen "not joined\n",'', 'Rmonitor';
            print_screen "   -> Resource is not joined to this node\n", "$Color_red"; 
            return; 
        } else {
            print_screen "Secondary", "$Color_blue bold";
        }
    }


    ##########################################################################
    ### check alive
    my @PAlive = lstat("$mars_dir/alive-$PName");
    if ( !$PAlive[9] ) { $PAlive[9] = 0 };
    my $PAlive =  time()- $PAlive[9] - $alife_timeout;
    print_screen ", System", '';
    if ( $PAlive > 1  ) {
        print_screen " unknown (last message before $PAlive sec) !!!", "$Color_red";
    } else {
        print_screen " alive", "$Color_green"; 
    }


    ##########################################################################
    ### check device
    # disk-device
    my $DiskDev = check_link "$mars_dir/$PRes/data-$PName";
    print_screen "\n\tDevice  : Disk-Device ";
    print_screen "$DiskDev", "$Color_blue";
    
    # resize
    my $ASize = check_link "$mars_dir/$PRes/actsize-$PName";
    if ( $PSize eq $ASize) {
        print_screen ", not enlarged";
    } else {
        print_screen ", resize active","$Color_red bold";
    }

    # mars-device
    my $MarsDev = "/dev/mars/$PDevice";
    my $Temp 	= "";
    if ( $PName eq $himself ) {			# himself
        if ( $PName eq $PStatus) {  		# himself=primary
            print_screen ", used as Mars-Device ";
            print_screen "$MarsDev", "$Color_blue";
            if ( stat( $MarsDev) ) {
                open my $fh, '<', '/proc/mounts' or die $!;
                $MarsDev = ( grep { /^$MarsDev / } <$fh> )[0];
                if ( $MarsDev ) {
                    $MarsDev = ( split / /, $MarsDev )[1];
                    print_screen "\n\t\t---> WORK: mounted as $MarsDev", "$Color_blue";
                } else {
                    print_screen "\n\t\t---> TODO: enable to mount", "$Color_green";
                }
            } else {
                print_screen "\n\t\t---> HINT: unable to mount, mars is starting or defective", "$Color_red";
            }
        } else {				# himself secondary
            if ( stat( $MarsDev) ) {
                open my $fh, '<', '/proc/mounts' or die $!;
                $MarsDev = ( grep { /^$MarsDev / } <$fh> )[0];
                if ( !$MarsDev ) {
                    print_screen "\n\t\t---> HINT: Mars-Device on Secondary available", "$Color_red";
                }
            }
        }
    }
    if ( $PSize ne $ASize) {
        print_screen "\n\t\t---> HINT: resizing used ($PSize != $ASize)","$Color_red";
    }
    $$ref_ResPartner++;


    ##########################################################################
    ### check sync
    ### sync - status
    my $PSyncsize = check_link "$mars_dir/$PRes/syncstatus-$PName";
    my $SStatus   = sprintf ("%.2f", ($PSyncsize / $PSize * 100));
    print_screen (sprintf "\n\tSync    : %s bytes (%.3fTB) synced = ", $PSyncsize, ( $PSyncsize/1024/1024/1024/1024));

    ### sync - speed
    my $SSpeed    = check_link "$mars_dir/$PRes/actual-$PName/sync_rate";
    $SSpeed       = sprintf ("%.3f", $SSpeed / 1024 / 1024);
    my $SEndTime  = ($PSize - $PSyncsize ) / 1024 / 1024 / 1024;
    if ( $SSpeed eq "0.000" ) {
        $SSpeed = "%";
    } else {
        $SEndTime = sprintf ("%.2f", $SEndTime / $SSpeed / 60);
        $SSpeed   = "%, at $SSpeed gb/s (done in $SEndTime min)";
    }

    ### sync - results
    if ( $SStatus < 100) {
        print_screen "$SStatus$SSpeed\n";
        print_screen "\t\t---> WORK: Sync in progress = ($SStatus% < 100.00%)\n", "$Color_blue";
    } else {
        print_screen "$SStatus$SSpeed\n", "$Color_green";
    }


    ##########################################################################
    ### TODO: work by resize ...


    ##########################################################################
    ### check logfile
    ### logfile - status
    my @PLogFile  = split (',', check_link "$mars_dir/$PRes/replay-$PName" );
    my @PLogLink  = split ("-", $PLogFile[0]);
    my $PLogName  = "$PLogLink[0]-$PLogLink[1]";
    my $PLogSize  = -s "$mars_dir/$PRes/$PLogFile[0]";
    if ( !$PLogFile[1] ) { 
        $PLogFile[1] = 0; $PLogFile[2] = 0; 
    }
    $PLogSize    = 0.0001 if (( !$PLogSize ) || ( $PLogSize eq 0 ));
    my $LogSpeed = check_link "$mars_dir/$PRes/actual-$PName/file_rate";
    $LogSpeed    = sprintf ("%.2f", $LogSpeed / 1024 / 1024);

    ### TODO:
    ### logfile - delay

    ### logfile - results
    print_screen (sprintf "\tLogfile : %s bytes (%.3fGB) in ", $PLogSize, ( $PLogSize/1024/1024/1024 ));
    print_screen "$PLogName", "$Color_green";
    print_screen " active";
    print_screen ", received with $LogSpeed gb/s" if ( $LogSpeed ne "0.00" );
    print_screen "\n";
    if ( $Ljoined eq "0" || $PLogSize eq "0.0001" ) {
        print_screen "\t\t---> WORK: Logfile wait for starting ...\n", "$Color_blue";
    }
    if ( ( $ref_AULogfile ) && !($PLogName eq $ref_AULogfile) ) {
        print_screen "\t\t---> HINT: Logfile Version not actual = ($PLogName ! $ref_AULogfile)\n", "$Color_red";
    }
	

    ##########################################################################
    ### check replay ...
    ### replay - status
    my $RStatus = sprintf("%.2f", ( $PLogFile[1] / $PLogSize * 100));
    $RStatus    = 0      if ( $Ljoined eq "0" || $PLogSize eq "1" ); 
    $RStatus    = 99.99  if ((  $PLogFile[1] ne $PLogSize ) && ( $RStatus eq "100.00" ));
    print_screen sprintf ("\tReplayed: %s bytes (%.3fGB) now replayed, Todo %d, completed ", 
        $PLogFile[1], ( $PLogFile[1]/1024/1024/1024 ), $PLogFile[2]);
        
    ### replay - speed
    my $RSpeed  = check_link "$mars_dir/$PRes/actual-$PName/replay_rate";
    $RSpeed     = sprintf ("%.2f", $RSpeed / 1024 / 1024);
    if ( $RSpeed eq "0.00" ) {
        $RSpeed = "%";
    } else {
        $RSpeed = "%, at $RSpeed gb/s";
    }

    ### replay - results
    if (( $RStatus < 1 ) && ( $PLogSize != 0.0001 )) {
        print_screen "$RStatus$RSpeed\n";
        print_screen "\t\t---> HINT: Replay not started, Logfile inactive = (Size: $PLogSize)\n", "$Color_red";

    } elsif (( $RStatus < 100 ) && ( $PLogSize != 0.0001 )) {
        print_screen "$RStatus$RSpeed\n";
        print_screen "\t\t---> WORK: Replay in progress = ($RStatus% < 100.00%)\n", "$Color_blue";

    } elsif ( $PLogFile[2] > 0 ) {
        $RStatus = sprintf("%.2f", ($PLogFile[1]-$PLogFile[2])/$PLogFile[1] * 100);
        print_screen "$RStatus$RSpeed\n", "$Color_red";

    } elsif ( $PLogSize = 0.0001 ) {
        $RStatus = "100.00";
        print_screen "$RStatus$RSpeed\n", "$Color_green";

    } else {
        print_screen "$RStatus% $RSpeed\n", "$Color_green";
    }

   ### replay - hints
   if ($PLogFile[2] != 0) {
       print_screen "\t\t---> WORK: Replay-Todo is actualy $PLogFile[2], ", "$Color_blue";
       if ( $PLogFile[2] < 0 ) {
           print_screen "replaying backwards ??? Check this !!!\n", "$Color_red";
       } elsif ( $PLogFile[2] > 0 ) {
           print_screen "mars it's working ...\n";
       } else {
           print_screen "replaying working unknown ... Check this !!!\n", "$Color_red";
       }
    }


    ##########################################################################
    ### check actual
    my $ActStatus  = check_link "$mars_dir/$PRes/actual-$PName/is-primary";
    my $ActDevice  = check_link "$mars_dir/$PRes/actual-$PName/device-$PDevice";
    print_screen "\tActual  : Status="; 
    if ( $ActStatus eq 1 ) {
        print_screen "Primary", "$Color_green";
        print_screen ", used Device="; 
        # hack for multiple linkversions
        if ( $ActDevice eq "off") {
            if ( convert_link "$mars_dir/$PRes/actual-$PName/device-$PDevice" eq "off" ) {
                print_screen "on", "$Color_red";
            } else { 
                print_screen "on", "$Color_green";
            }
        } else {
            print_screen "on", "$Color_green";
        }
    } else {
        print_screen "Secondary", "$Color_green";
        print_screen ", Syncstatus="; 
        convert_link "$mars_dir/$PRes/actual-$PName/copy-syncstatus-$PName";
        print_screen ", Logfileupdate="; 
        convert_link "$mars_dir/$PRes/actual-$PName/logfile-update";
    }
    print_screen ", Attached="; 
    convert_link "$mars_dir/$PRes/actual-$PName/is-attached";
    print_screen "\n";

    ##########################################################################
    ### check switches
    my $SWStatus;
    print_screen "\tSwitches: Attach="; 
    if ( readlink "$mars_dir/$PRes/todo-$PName/attach" eq 1 ) {		### Use of uninitialized value in string
        print_screen "on", "$Color_green";
    } else {
        print_screen "off", "$Color_red";
    }
    print_screen " [masked:" if ( $ActStatus eq 1 );
    print_screen " Connect="; 
    if ( readlink "$mars_dir/$PRes/todo-$PName/connect" eq 1 ) {	### Use of uninitialized value in string
        print_screen "on", "$Color_green";
    } else {
        print_screen "off", "$Color_red";
    }
    print_screen " Sync=";
    if ( readlink "$mars_dir/$PRes/todo-$PName/sync" eq 1 ) {		### Use of uninitialized value in string
        print_screen "on", "$Color_green";
    } else {
        print_screen "off", "$Color_red";
    }
    print_screen " AllowReplay=" ;
    if ( readlink "$mars_dir/$PRes/todo-$PName/allow-replay" eq 1 ) {	### Use of uninitialized value in string
        print_screen "on", "$Color_green";
    } else {
        print_screen "off", "$Color_red";
    }
    print_screen "]" if ( $ActStatus eq 1 );
    print_screen "\n";

    return $PLogName;
}


#########################################################################################
### check ressources
sub check_ressource {
    opendir my $dirhandle, $mars_dir or die "Cannot open $mars_dir: $!";
    my @resources = grep { /^res/ && -d "$mars_dir/$_" } readdir $dirhandle;
    if ( !@resources ) {
        print_screen "---> HINT: no resources found\n", "$Color_red";
        next;
    }
	
    ### read resources
    foreach my $res (@resources) {
        my $ResPartner  = 0;
        my $res_name    = $res;
        $res_name       =~ s/^resource-//;
        if ( $params->{'resource'} ) {
            if (!( $params->{'resource'} eq $res_name)) {
                next;
            }
        }
        my $res_size    = check_link "$mars_dir/$res/size";
        if ( $res_size eq 0 ) { $res_size = 1 };
        my $res_tbsize  = ( $res_size) / 1024 / 1024 /1024 / 1024;
        my $res_master  = check_link "$mars_dir/$res/primary";
        if ( $res_master eq 0 ) { $res_master = "unknown" };
        print_screen sprintf("-> check resource %s, with %.3fTB, Primary Node is %s\n", $res_name, $res_tbsize, $res_master), 'bold';
        
        ### him self
        my $himselfip = check_link "$mars_dir/ips/ip-$himself";
        print_screen "   -> local Node ($himself [$himselfip]) as ",'bold';
        my $ActualUsedLogfile = display_partner(
            ressource		=> $res,
            nodename		=> $himself,
            ressource_size	=> $res_size,
            res_partner		=> \$ResPartner,
            res_AULogfile   	=> "",
        );
        # end him self


        ### joined (und nicht monitor)...
        if ( $ResPartner eq 1 ) {
            ### partners
            opendir my $server_dh, "$mars_dir/$res" or die "Cannot open $mars_dir/$res: $!";
            my @servers = grep { /^data/ && readlink "$mars_dir/$res/$_" } readdir $server_dh;
            @servers    = sort (@servers);
            foreach my $partner (@servers) {
                $partner  =~ s/^data-//;
                if ( $partner eq $himself ) { next; }
                $himselfip = check_link "$mars_dir/ips/ip-$partner";
                print_screen "   -> remote Node ($partner [$himselfip]) as ", 'bold';
                display_partner(
                    ressource		=> $res,
                    nodename		=> $partner,
                    ressource_size	=> $res_size,
                    res_partner		=> \$ResPartner,
                    res_AULogfile   	=> $ActualUsedLogfile,
                    );
            }
        } # end joined

	
        ### modus
        if ( $ResPartner eq 0 ) { 
            print_screen "   -> modus for resource $res_name is remote ($ResPartner nodes)\n",'bold'; 
        } elsif ( $ResPartner eq 1 ) { 
            print_screen "   -> modus for resource $res_name is standalone ($ResPartner node)\n",'bold'; 
        } else {
            print_screen "   -> modus for resource $res_name is clustered ($ResPartner nodes)\n ",'bold'; 
        }
	
        
        ### resources history
        if ( $params->{'history'} ) {
            check_logfile( $res, $ResPartner );
        }


        ### check resources debug
        if ($params->{'debug'}) {
            print_screen "   -> Debug for $res\n", 'bold'; 
            my $debug_res;
            ### TODO: small hack, read 3 files ...
            $debug_res = check_debugfile("$res", "2.warn");  print_screen "$debug_res" if ( $debug_res );
            $debug_res = check_debugfile("$res", "3.error"); print_screen "$debug_res" if ( $debug_res );
            $debug_res = check_debugfile("$res", "4.fatal"); print_screen "$debug_res" if ( $debug_res );
        }

    } # end foreach

}
#########################################################################################
###
sub check_logfile {
    my $LResource 	= shift;
    my $LPartner  	= shift;
    my $LastVersionNr	= "0";
    print_screen "  -> History Replay/Status\n", 'bold';


    ### search all version's
    my @Version = <$mars_dir/$LResource/version-*>;
    foreach my $Version (@Version) {
        my $VersionNr	= $Version;
        $VersionNr      =~ s/^.*version-([0-9]+)-.*$/$1/;
        if ( "$LastVersionNr" eq "$VersionNr" ) {
            next;	# same Versionnr -> next
        } else {
            $LastVersionNr = $VersionNr;
        }
        print_screen "\t$Gls\n", "$Color_red";
        print_screen "\t$Gkr$Gab", "$Color_red";
        print_screen "Vers.$VersionNr", "$Color_blue";
        
        ### check logfile
        my @LogFile = <$mars_dir/$LResource/log-$VersionNr-*>;
        my $LogFile = $LogFile[0];
        my $LogSize = 0;
        my $LogHost = "";

        if ( $LogFile) { 
            ### found logfile    
            $LogSize = -s "$LogFile";
            $LogHost = $LogFile;
            $LogHost =~ s/.*log-([0-9]+)-//;
            if ( !$LogSize ) { $LogSize = 0; }
                ### logfile stat
                my @LogStat  = stat ( $LogFile );
                $LogStat[10] = gmtime($LogStat[10]);
                ### quickfix times ...
                if ($LogStat[9] > $LogStat[8]) {
                    $LogStat[9] = $LogStat[9] - $LogStat[8];
                } else {
                    $LogStat[9] = $LogStat[8] - $LogStat[9];
                }

                print_screen " $Gab", "$Color_red";
#                print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
                print_screen sprintf("Logfile Size: $LogSize bytes (%.3fGB) by %s from %s, include hypothetically %s sec\n", ($LogSize /1024/1024/1024), $LogHost, $LogStat[10], $LogStat[9]), "$Color_blue";
        } else {
            ### not found logfile
            print_screen " $Gab", "$Color_red";
#            print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
            print_screen "old Version, Logfile is deleted ...\n", "$Color_blue";
        }
                                                                                                                                                                                                   
        ### check sources
        my $VersionFileCount  = 0;
        my $VersionErrorSize  = 0;
        my $VersionErrorChk   = 0;
        my $VersionLastChk    = 0;
        my @VersionFile       = <$mars_dir/$LResource/version-$VersionNr*>;
        my $VersionNode	      = "";
        foreach my $VersionFile (@VersionFile) {
            my @VersionDetail = check_link "$VersionFile";
            @VersionDetail    = split (',|:', "@VersionDetail" );

            my $VersionSource = $VersionFile;
            $VersionSource    =~ s/.*[0-9]-//;

            ### add counter for node-check
            $VersionFileCount++;

            ### 0 chksum log now / 1 name log now / 2 size log now / 3 chksum log old / 4 name log old / 5 size log old
            if ( $VersionSource eq $LogHost ) {
                # primary
                if ( ($VersionFileCount eq $LPartner) || ($VersionFileCount > 1) ) {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gao$Gab", "$Color_red";
                } else {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gao$Gab", "$Color_red";
                }
                $VersionNode = "Primary  \t  ";

            } elsif ( $LogHost eq "" ) {
                # none
                if ( $VersionFileCount eq $LPartner ) {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gao$Gab", "$Color_red";
                } else {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gkr$Gab", "$Color_red";
                }

            } else {
               # secondary
                if ( $VersionFileCount eq $LPartner ) {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr$Gfr      $Gao$Gab", "$Color_red";
                } elsif ( $VersionFileCount > 1 ) {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gls$Gfr$Gkr$Gab", "$Color_red";
                } else {
                    print_screen "\t$Gls$Gfr$Gls$Gfr$Gfr     $Gls$Gfr$Gau$Gab", "$Color_red";
                }
                $VersionNode = "Secondary";
            };
            print_screen sprintf ("$VersionNode $VersionSource  \tCheck: $VersionDetail[0]  \tSize: $VersionDetail[2] bytes (%.3fGB)", $VersionDetail[2] / 1024 / 1024 / 1024 );
            

            if ( $LogFile ) {
                ### size
                $LogSize = $VersionDetail[2] if ( $LogSize < $VersionDetail[2] ); # hack, wenn secondary nicht das ganze log hat
                if ( $VersionDetail[2] eq 0 ) {
                    print_screen " ---> replay waiting for start", "$Color_green";
                } elsif ( $LogSize eq $VersionDetail[2] ) {
                    print_screen " ---> replay OK", "$Color_green";
                } else {
                    my $ReplayTodo = ($LogSize - $VersionDetail[2]) / 1024 / 1024 / 1024;
                    print_screen sprintf (" ---> replay incomplete (Todo %.3fGB)",$ReplayTodo), "$Color_red";
                    $VersionErrorSize = 1;
                }
                ### chksum
                if ( $VersionErrorSize eq 1 ) {
                    print_screen "\n";
                    $VersionErrorChk = 1;
                } elsif ( $VersionLastChk eq $VersionDetail[0] || $VersionLastChk eq 0 ) {
                    print_screen ", verify OK\n", "$Color_green";
                } else {
                    ### TODO: primary first system ?
                    print_screen ", verify failed\n", "$Color_red";
                    $VersionErrorChk = 1;
                }

                ### TODO: failed chksum ?
                $VersionLastChk = $VersionDetail[0]; # save for next foreach
            } else {
                ### no logfile found
                print_screen "\t(no longer available logfile)\n";
            }
            
            ### check delete infos
            my @DeleteFiles = <$mars_dir/todo-global/delete-*>;
            foreach my $DeleteFiles (@DeleteFiles) {
                if (( !$DeleteFiles ) || ( !(readlink $DeleteFiles) )) {
                    $DeleteFiles = "n/a";
                } else {
                    $DeleteFiles = basename (readlink $DeleteFiles);
                }
                if ( $LogFile && $DeleteFiles eq $LogFile ) {
                    print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
                    print_screen "\t\t---> TODO: LogFile ist marked for delete ($DeleteFiles).\n", "$Color_green";
                }
            }
            
        } # end foreach $VersionFiles
        
        ### same checks
        if ( $VersionFileCount ne $LPartner ) {
            print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
            print_screen "HINT: Count of Logfiles different = (Cluster has $LPartner Nodes, but only find $VersionFileCount Node)\n", "$Color_red";
        }
        
        if ( $VersionErrorSize eq 1 && $VersionErrorChk eq 1)  { 
#            print_screen "\t$Gls$Gfr$Gao$Gab\n", "$Color_red";
        } elsif ( $VersionErrorSize eq 1 ) {
            print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
            print_screen "HINT: Logfiles has not equal size and same Checksums, ups ... \n","$Color_red";
        } elsif ( $VersionErrorSize ne 1 && $VersionErrorChk eq 1 ) {
            print_screen "\t$Gls$Gfr$Gkr$Gab", "$Color_red";
            print_screen "HINT: Logfiles has not equal Checksums and same size, check for SPLIT-BRAIN\n","$Color_red";
        }

        if ( `ls -l $mars_dir/$LResource/replay-* | grep $VersionNr` ) {
            print_screen "\t$Gls$Gfr$Gao$Gab", "$Color_red";
            print_screen "WORK: Version are actual and used. ", "$Color_blue";
            if ( $VersionErrorSize ne 1 && $VersionErrorChk ne 1) {
                print_screen "Wait for start replay ...\n", "$Color_green";
            } else {
                print_screen "Replay in progress ...\n", "$Color_green";
            }
        } elsif ( !$LogFile ) {
            print_screen "\t$Gls$Gfr$Gao$Gab", "$Color_red";
            print_screen "WORK: Version is deleted the next log-rotate ...\n", "$Color_blue";
        } elsif ( !`ls -l $mars_dir/$LResource/replay-* | grep $VersionNr` && $LogFile ) {
            print_screen "\t$Gls$Gfr$Gao$Gab", "$Color_red";
            print_screen "TODO: Logfiles has all equal Sizes and Checksums, can be deleted ...\n", "$Color_green";
        } else {
            print "ups ??";
        }

    } # end foreach $Version
}


#########################################################################################
### check debug-files
sub check_debugfile {
    ### TODO: Fix Level / diaplying
    my $debug_dir   = shift;
    my $debug_level = shift;
    my $debug_file  = "$mars_dir/$debug_dir/$debug_level.status";
    if ( open  (MARS_DEBUG, "< $debug_file") ) {
        my $mars_debug = "";
        while ( <MARS_DEBUG> ) {
            if ( m/^(\d+\.\d+)/ ) {
                s/^(\d+\.\d+)/strftime("%a %b %e %H:%M:%S %Y:", localtime($1))/e;
            } else {
                $_ = localtime(0) . ': ' . $_;
            }
            s/MARS_DEBUG\s+//;
            $mars_debug = "$mars_debug\t$_";
        }
        close MARS_DEBUG;
        return "$mars_debug"  if ( $mars_debug ne "");
    } 
}       

#########################################################################################
### info version
sub info_version {
    ### module
    my %mars_info;
    open ( my $lsmod_handle,'-|','lsmod | grep mars' ) || die "blub ... $!";
    if (!<$lsmod_handle>) {
        print_screen "Module Mars not running\n","$Color_red";
        sleep(10);
        next;
    }
    open ( my $modinfo_handle, '-|', 'modinfo mars' ) || die "cannot run modinfo mars: $!";
    while ( my $line = <$modinfo_handle> ) {
        chomp $line;
        my ( $key, $value) = split /: +/, $line;
        if ( $value) {
            $mars_info{$key} = $value;
        }
    }
    if ( $mars_info{author} eq "") {
        print_screen "Module Mars not running\n","$Color_red";
        next;
    }
    
    ### status
    print_screen "MARS Status - $himself, $version", "$Color_blue";
    if ( $params->{'resource'}  ) { print_screen ", Ressource: $params->{'resource'}", "$Color_blue"; }
    print_screen "\n";
    
    ### marsadm
    my $MAVersion = qx"marsadm version";
    print_screen "MARS Admin  - $MAVersion", "$Color_blue";
    
    ### module
    print_screen "MARS Module - $mars_info{version}\n", "$Color_blue";

    ### kernel
    my $KVersion = '/proc/version';
    open my $Kfh, '<', "$KVersion" or die $!;
    $KVersion = ( grep { /^Linux/ } <$Kfh> )[0];
    $KVersion = ( split / /, $KVersion )[2];
    print_screen "MARS Kernel - $KVersion\n", "$Color_blue";
    
    print_screen "-------------------------------------------------------------------------------\n";
}


#########################################################################################
### diskfull
sub check_diskfull {
    my @diskfull = glob("$mars_dir/rest-space-*");
    my $diskfull_mars = "";
    print_screen "-> Cluster Diskspace:", 'bold';
    if ( @diskfull ) { 
        foreach ( @diskfull ) {
            my $diskfull_space     = check_link "$_";                        
            my $diskfull_system = $_;                
            $diskfull_system    =~ s!/mars/rest-space-!!;
            if ( $diskfull_space < 1 ) {
                $diskfull_space = sprintf ("%.2f", $diskfull_space / 1024 );
                if ( $diskfull_system eq $himself ) {
                    print_screen "\n\t-> HINT: Local Partition $mars_dir full ($diskfull_space kb Limit) !!! mars is stopping !!!\n->", "$Color_red bold";
                    $diskfull_mars = "$diskfull_mars,$diskfull_system";
                } else {
                    print_screen "\n\t-> HINT: Remotesystem $diskfull_system have mars-disk full ($diskfull_space kb Limit) !!!\n->", "$Color_red bold";
                    $diskfull_mars = "$diskfull_mars,$diskfull_system";
                }
            }
       }
    }
    ### TODO: /0
    if ( !$diskfull_mars ) {
        print_screen " smoothly ", "$Color_green";
    }
}

#########################################################################################
### emergency
sub check_jammed {
    my $jammed = check_link "$mars_dir/emergency-$himself";
    print_screen " Transaktions:", 'bold';
    if ( !$jammed ) {
        print_screen " smoothly ", "$Color_green";
    } else {
        print_screen " and Replication not running !!!\n-> ", "$Color_red";
    }            
}

#########################################################################################
### connects
sub check_connects {
    my $jammed = check_link "$mars_dir/emergency-$himself";
    print_screen " Connects:", 'bold';
    if ( !$jammed ) {
        print_screen " TODO ", "$Color_green";
    } else {
        print_screen " TODO !!!\n", "$Color_red";
    }            
}

#########################################################################################
### synclimit
sub check_synclimit {
    my $synclimit;
    if ( open (MARS_LIMIT, "< /proc/sys/mars/sync_limit") ) {
        while (<MARS_LIMIT>) {
            $synclimit .= $_;
            $synclimit =~ s/[\n\t]//g;
        }
        close MARS_LIMIT;
    }
    print_screen " Synclimit:", 'bold';
    if ( !$synclimit ) {
        print_screen " smoothly ", "$Color_green";
    } else {
        print_screen " set to $synclimit !!!\n", "$Color_red";
    }            
}

#########################################################################################
### limit's auslesen ...
sub check_limit {
    my $LimitText    = shift; 		# sol-text
    my $LimitSolVar  = shift; 		# sol-filename
    my $LimitSolEin  = shift; 		# sol-einheit
    my $LimitIstVar  = shift; 		# ist-filename
    my $LimitIstEin  = shift; 		# ist-einheit
    $LimitSolVar = "" if (!$LimitSolVar); 
    $LimitIstVar = "" if (!$LimitIstVar); 
    $LimitSolEin = "" if (!$LimitSolEin);
    $LimitIstEin = "" if (!$LimitIstEin);

    ### soll
    my $mars_limit_sol;
    if ( open (MARS_LIMIT, "< /proc/sys/mars/$LimitSolVar") ) {
        while (<MARS_LIMIT>) {
            $mars_limit_sol .= $_;
            $mars_limit_sol =~ s/[\n\t]//g;
        }
        close MARS_LIMIT;
    }
    $mars_limit_sol = 0 if ( !$mars_limit_sol);

    ### ist
    my $mars_limit_ist;
    if ( open (MARS_LIMIT, "< /proc/sys/mars/$LimitIstVar") ) {
        while (<MARS_LIMIT>) {
            $mars_limit_ist .= $_;
            $mars_limit_ist =~ s/[\n\t]//g;
        }
        close MARS_LIMIT;
     }
     $mars_limit_ist = 0 if ( !$mars_limit_ist);
    
    ### presently results
    print_screen "$LimitText ", 'bold';    
    if ( ($LimitSolVar) && !($LimitIstVar) ) {
        ### sol & lamport_clock
        if ( $LimitSolVar eq "lamport_clock" ) {
            my $C_Time = $mars_limit_sol;
            $C_Time    =~ s/CURRENT_TIME=//;
            $C_Time    =~ s/lamport_now=.*//;
            my $L_Time = $mars_limit_sol;        
            $L_Time    =~ s/.*lamport_now=//; 
            $mars_limit_sol = sprintf("%.2f", $C_Time - $L_Time);
            print_screen "$mars_limit_sol $LimitSolEin,";
        ### restliches
        } elsif ( $mars_limit_sol < 1 ) {
            print_screen "is now unsed,", "$Color_green";
        } else {
            print_screen "is set to ";
            print_screen "$mars_limit_sol $LimitSolEin,", "$Color_red";
        }
    } elsif ( !($LimitSolVar) && ($LimitIstVar) ) {
        ### only ist
        print_screen "is actualy ";

        if ( $mars_limit_ist < 1 ) {
            if ( $LimitIstEin eq "on/off" ) {
                print_screen "off,", "$Color_green";
            } else {
                print_screen "null", "$Color_green";
            }
        } else {
            if ( $LimitIstEin eq "on/off" ) {
                print_screen "on,", "$Color_red";
            } else {
                print_screen "$mars_limit_ist $LimitIstEin,", "$Color_red";
            }
        }
# TODO fixen !
#    } elsif ( ($LimitSolVar) && ($LimitIstVar) && ($mars_limit_sol < 1) ) {    
#        ### sol & ist = 0
#        print_screen "is actualy unused(X),";
    } else {
        ### sol & ist / rest ...
        print_screen "is set to ";
        print_screen "$mars_limit_sol $LimitSolEin", "$Color_red";
        print_screen ", actualy used ";
        print_screen "$mars_limit_ist $LimitIstEin,", "$Color_red";
    }
        
}


##############################################################################
### mars-system
sub check_systemstatus {
    print_screen "---> Systemdata <---\n", "$Color_blue bold";

    ### text / sol-file / sol-einheit / ist-file / ist-einheit
    check_limit "-> AVG Limit",			"loadavg_limit",					"loadavg";
    print "\n";
    check_limit "-> Memory Limit",		"mem_limit_percent", 					"%",	"mem_used_raw_kb", 				"kb";
    print "\n";
    check_limit "-> Traffic Limit Client",	"traffic_tuning/client_role_traffic_ratelimit_kb",      "kb/s",	"traffic_tuning/client_role_traffic_rate_kb",	"kb/s";
    print "\n";
    check_limit "-> Traffic Limit Server",	"traffic_tuning/server_role_traffic_ratelimit_kb",      "kb/s",	"traffic_tuning/server_role_traffic_rate_kb",	"kb/s";
    print "\n";
    check_limit "-> Traffic Writeback", 	"traffic_tuning/writeback_ratelimit_kb", 		"kb/s",	"traffic_tuning/writeback_rate_kb", 		"kb/s";
    print "\n";
    check_limit "-> I/O Writeback", 		"io_tuning/writeback_ratelimit_kb", 			"kb/s",	"io_tuning/writeback_rate_kb", 			"kb/s";
    print "\n";
    check_limit "-> Server-IO Limit", 		"tuning/server_io_limit_kb", 				"kb/s", "tuning/server_io_rate_kb", 			"kb/s";
    check_limit " Flying IO",			"",							"",	"io_flying_count";
    print "\n";
    check_limit "-> Copy Read: Prio",		"copy_read_prio",					"";
    check_limit " Flying IO",			"",							"",	"copy_read_max_fly";
    print "\n";	
    check_limit "-> Copy Write: Prio",		"copy_write_prio",					"";
    check_limit " Flying IO",			"",							"",	"copy_write_max_fly";
    print "\n";
    check_limit "-> LoggerMemory",		"",							"",	"logger_mem_used_kb",				"kb";
    print "\n";
    check_limit "-> FreeSpaceLimit LogRotate",	"logrot_auto_gb",					"gb";
    print "\n";
    check_limit "-> Network-IO-Timeout",	"network_io_timeout",					"sec";
    print "\n";
    check_limit "-> Clear Page Cache",      	"",                             			"",     "mapfree_period_sec",				"sec";
    print "\n";
    check_limit "-> Statusfile Rollover",	"",							"",	"statusfiles_rollover_sec",			"sec";
    print "\n";
    check_limit "-> Modus: Fast Full Sync", 	"",                             			"",     "do_fast_fullsync",				"on/off";
    check_limit " AIO Sync",       		"",                             			"",     "aio_sync_mode",				"on/off";
    check_limit " Delay say Overflow",		"",							"",	"delay_say_on_overflow",			"on/off";
    check_limit " Emergency",       	"",                             				"",     "mars_emergency_mode",				"on/off";
    check_limit " Logger Resume",       	"",                             			"",     "logger_resume",				"on/off";
    print "\n";
    check_limit "-> LamportClockDifferenz",	"lamport_clock",					"sec";
    check_limit " Mars Port",       		"mars_port",                    			""; 
    print "\n";

    my $mars_disk_space = `df '$mars_dir' | grep '$mars_dir'| awk '{print \$2}'`;
    $mars_disk_space = sprintf("%01.2f", $mars_disk_space / 1024);
    check_limit "-> Free-Space-Limit on /mars", "required_free_space_1_gb", 	"mb (actualy $mars_disk_space mb used)";
    print "\n";
}

##############################################################################
### main loop ...
while(1) {
    my $dateFormat = Date::Language->new('English');

    ### version only
    if ( $params->{version} || $params->{v}) {
        info_version;
        exit 0;
    }
        

    ##########################################################################
    ### main run
    print $clearscreen;
    print "\nNOTE !!!\n********\nThe author does not guarantee this development-test-alpha-pre-beta-version, it is untested and certainly not fully functional. Use at your own risk ;)\n\n";


    ##########################################################################
    ### mars-tree-version
    my @MarsTreeVersion = <$mars_dir/tree-*>;
    foreach my $MarsTreeVersion (@MarsTreeVersion) {
       if (check_link "$MarsTreeVersion" ne $MarsTreeVer ) {
           print "*** Sorry, unknown Tree-Version of Mars unknown\n";
           exit 1;
       }
    }


    ##########################################################################
    ### read mars infos
    info_version;


    ##########################################################################
    ### check system limits
    if ( $params->{'system'} ) {
        check_systemstatus;
    }
    
    
    ##########################################################################
    ### check system params
    check_diskfull;
    check_jammed;
    check_connects;
    check_synclimit;
    
    ### check resources
    print_screen "\n---> Resources <---\n", "$Color_blue bold";
    check_ressource;
	
	
    ##########################################################################
     ### check global debug
        if ($params->{'debug'}) {
        print_screen "---> Debug <---\n", "$Color_blue bold";

            print_screen "-> Main-Debug:\n", "$Color_red"; 
            my $debug_res;
            $debug_res = check_debugfile("", "2.warn");  print_screen "$debug_res" if ( $debug_res );
            $debug_res = check_debugfile("", "3.error"); print_screen "$debug_res" if ( $debug_res );
            $debug_res = check_debugfile("", "4.fatal"); print_screen "$debug_res" if ( $debug_res );
        }


    ##########################################################################
    ### end, next loop
    print color 'reset';
    exit if (not $params->{'interval'});
    sleep($params->{'interval'});

}
exit;