# syncdb.pl
#
# Copyright (c) 1993, 1994, 1995, 1996, 1997, 1998, 1999 by RIPE NCC
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted,
# provided that the above copyright notice appear in all copies and that
# both that copyright notice and this permission notice appear in
# supporting documentation, and that the name of the author not be
# used in advertising or publicity pertaining to distribution of the
# software without specific, written prior permission.
#
# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
# AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
# AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# $Id: syncdb.pl,v 2.17 1999/10/06 16:41:21 marek Exp $
#
#	$RCSfile: syncdb.pl,v $
#	$Revision: 2.17 $
#	$Author: marek $
#	$Date: 1999/10/06 16:41:21 $

# This is a client that will update objects read from a file directly
# in the database, without interference of updated.
# It will update the object in the database that is linked to the
# source field of the object, so better make sure you have a source field
# and that it has a database associated to it ...

# System libraries

use Getopt::Std qw();

# RIPE DB libraries
require "adderror.pl";
require "cldb.pl";
require "dbadd.pl";
require "dbclose.pl";
require "dbopen.pl";
require "enread.pl";
require "enwrite.pl";
require "misc.pl";
require "rconf.pl";
require "dpr.pl";

require "syslog.pl";     # nothing to do with the system 'syslog.pl'
require "whoisqry.pl";

## parse command line

Getopt::Std::getopts('h:p:s:u:p:Vb');

$::WHOIS_HOST = $::RIPEConfig{WHOISHOST};
$::WHOIS_HOST = $opt_h if $opt_h;

$::WHOIS_PORT = '';
$::WHOIS_PORT = $opt_p if $opt_p;

$Main::is_update=1;

# check command line and maybe print usage 
if ((!$opt_s) || (($opt_u) && ($opt_u!~ /^\d+\:\d+$/))) {

  print <<"EOF";
    
Usage: $PROGRAMNAME [ -h hostname ] [ -p portno ] -s source [ -u period:delay ]
   
Where:

-h hostname      - hostname of server that provides updates.
                   Default - $::RIPEConfig{WHOISHOST}
-p portno        - port number of server that provides updates.
                   Default - look up service "whois" in /etc/services
-s source        - which source to query for
-u period:delay  - the program will run for 'period' seconds and
                   will do a query every 'delay' seconds
-V               - verbose output
-b               - prints a lot of useless debugging information
EOF

  exit 1;
   
}

# SYNCFASTMODE
# 
# Poll the given serial number file to find out when new updates are 
# available. Then use normal 'whois -g' query to fetch updates.
# 
# opt_u works exactly the same but checks the serial file instead of the
# server for changes. period part of $opt_u is not used.
#
# $SYNCFASTMODE="/ncc/dbase/log/serials/".$opt_s.$CURRENTEXTENSION;

#
# Read config file from RIPEDBCNF, or set to default.

{
  my $conffile;

  $conffile = $ENV{RIPEDBCNF};
  $conffile = $::RIPEConfig{DEFCONFIG} unless $conffile;
  &rconf($conffile);
}

#
# run on low priority, whois server queries have an
# higher priority

&lower_priority();

%entry=();

$locklink=$LOCKDIR.$PROGRAMNAME.".".$opt_s.".RUNNING";

$tmpdata=$TMPDIR."/".$PROGRAMNAME.".".$$;

#
# this handler is to stop updating in a as decent as possible
# when errors occur

$STOPUPDATING=0;

sub stopupdating {
    
    $STOPUPDATING=$_[0];
}

#
# this handler is for the the normal timeout as
# defined by $opt_u

$TIMEOUT=0;

sub timeouthandler {

    $TIMEOUT=1;
    
}

$PROBLEMS='(ERROR|Warning|Closing\s+connection)';

#
# exit syncdbase in a clean way

sub exitsyncdbase {
   local($returncode)=@_;
   
   if (!$returncode) {
      unlink($tmpdata) or 
	warn "couldn't remove temporary datafile: $tmpdata\n";
   }
   
   unlink($locklink) || warn "couldn\'t remove lockfile: $locklink\n";
   
   &fatalerror($returncode) if ($returncode!~ /^\d+$/);
   
   exit(1) if (($returncode) && ($returncode!~ /^\d+$/));
   
   exit $returncode;
   
}

sub skiperrorserial {
   local(*entry,$action,$source,$msg)=@_;
   
   $entry{"pn"}="problem found" if (!&entype(%entry));
   $entry{"so"}=$source if (!$entry{"so"});
   
   &adderror(*entry,$msg);
   
   &writeseriallog($action,*entry);
     
}

#
# doaction ( )
#
# does all the work.
#

sub doaction {

    local(*entry,$type,$action,$source,$serial) = @_;
    
#    &dpr("doaction($action,$serial) called\n");
    
    local($returncode)=0;
    
    #
    # open database file associated with "so" for writing

    local($mirrordb)='mirrordb';
    local(%mirrordb,@mirrordb);
    
#    &dpr("doaction opening database\n");

#    local($type)=&entype(%entry);

    if (&dbopen(*mirrordb, *entry, 1)) {
       # Open classless database

#       &dpr("doaction opening classless db\n");
       &dbclopen(*entry,1) if ($CLASSLESSDBS{$type});

       # Looks like we have some locking problem, so let's
       # lock the thing before we do anything else ...

#       &dpr("doaction locking databases\n");
		
       # do we delete or not ?

       if ($STOPUPDATING) {
          
          $returncode="$PROGRAMNAME (cleanly) terminated by signal ($STOPUPDATING)";
       
       }
       else {
       
          if ($action=~ /^$DELETEACTION$/i) {

#             &dpr("doaction deleting entry ($entry{$type})\n");
             $dbstat = &dbdel(*mirrordb, *entry, $type, $DELETEOPTION | $NOCHECKSOPTION);

          }
          elsif ($action=~ /^$ADDACTION$/i) {
          
#             &dpr("doaction adding entry ($entry{$type})\n");
             $dbstat = &dbadd(*mirrordb, *entry, $type, 0);
          
          }
          
       }  

       #
       # noop, just print stat if verbose but always print errors!!!

#       &dpr("doaction checking stat\n");

       if ($dbstat == $E_NOOP) {
          print STDERR "ERROR, incomingserial ($action: $entry{$type}): ", $LOGFILE{"SERIALINCOMINGDIR"}, $serial, " [NOOP]\n\n";
          &skiperrorserial(*entry,$action,$source,eval $MESSAGE[$dbstat]);
       }
       elsif ($dbstat!=$OK) {
          print STDERR "ERROR, incomingserial ($action: $entry{$type}): ", $LOGFILE{"SERIALINCOMINGDIR"}, $serial, " ", eval $MESSAGE[$dbstat], "\n\n";
          &skiperrorserial(*entry,$action,$source,eval $MESSAGE[$dbstat]);
       }
       else {
#          &dpr("OK, ($action: $entry{$type}) serial: $serial\n");
       }

       &dbclose(*mirrordb);
       &dbclclose() if ($CLASSLESSDBS{$type});

    }
    
    return $returncode;

}

sub UpdateVersionOne {
   local($input,$line,$opt_s,*from,$to,$period)=@_;

   local(%entry,%oldentry);
   local($first,$last,$source,$type,$oldtype);
   local($i,$oldi,$action,$oldaction);
   
   local($returncode)=0;
   
   $line=~ /^\%START\s+Version:\s*\d+\s+(\S+)\s+(\d+)\-(\d+)\s*$/;
   
   $source=$1;
   $first=$2;
   $last=$3;
   
   my $basename = $LOGFILE{"SERIALINCOMINGDIR"}.$source;
   
#   &dpr("basename: $basename source: $source first: $first last: $last\n");
      
   $i=$first;
   $line=<$input>;
   
   &exitsyncdbase("got other start serial ($first) then expected ($from)") if ($first!=$from);
   &exitsyncdbase("got other end serial ($last) then expected ($to)") if (($to!~ /^LAST$/i) && ($last!=$to));
   &exitsyncdbase("got source ($source) then expected ($opt_s)") if ($source ne $opt_s); 
   
   
   while ((!$STOPUPDATING) &&
          ($i<=$last) &&
          ($line=<$input>) &&
          ($line!~ /^\s*$/) &&
          ($line!~ /^\%END/)) {
   
#      &dpr("update: $basename.$i\n");
      
      if ($line=~ /^$ADDACTION$/i) {
         $action=$ADDACTION;
      }
      elsif ($line=~ /^$DELETEACTION$/i) {
         $action=$DELETEACTION;
      }
      else {
         return $line;
      }
      
      # will write update to its own file if a directory is given in config
      # file
      if (defined $::LOGFILE{"SERIALINCOMINGDIR"}) {
	open(OUTP,">$basename.$i");
	print OUTP $line;
      }

      $line=<$input>;
      
      return $line if ($line!~ /^\s*$/);
      
      $type=&enread($input, *entry, -1);
      
      return $type if (!defined($OBJATSQ{$type}));
      
      if (defined $::LOGFILE{"SERIALINCOMINGDIR"}) {
	&enwrite(OUTP,*entry,0,1,0);
	close(OUTP);
      }

      &doaction(*oldentry,$oldtype,$oldaction,$source,$oldi) if ($i!=$first);
   
      %oldentry=%entry;
      $oldtype=$type;
      $oldaction=$action;
      $oldi=$i;
   
      $i++;
      
   }
   
   # print STDERR "last line: $line";

   # handy for debugging!

   local($offset)=tell(TMPFILE);

   if ($line=~ /^\s*$/) {
      $line=<TMPFILE>;
   }

   if ($STOPUPDATING) {
      
      $returncode="$PROGRAMNAME (cleanly) terminated by signal ($STOPUPDATING)";
      
      close(TMPFILE);
      
   }
   elsif ($line!~ /^\%END/) {
   
      local(@lines)=();
      local($oldline)=$line;
      
      if (($line=~ /^\d+$/) && ($offset>$line)) {
         
         seek(TMPFILE, $line, 0);
         
         while (($line=<TMPFILE>) && ($offset<=tell(TMPFILE))) {
            push(@lines, $line);
         }
         
      }
      else {
         @lines=($line);
      }
   
      if ((@lines=grep((/^\%/) && (/$PROBLEMS/i), @lines))) {
      
         if (grep(/ERROR/i, @lines)) {
         
            $returncode="Error found: ".join("", @lines);
            
         }
         else {
            
            print STDERR "Warning (from raw input data): ".join("", @lines);
            
         }
      
      }
      else {
      
         $returncode="Error in output from whois ($basename.$i)\nnear offset: $offset line: ".$oldline."\n";
         
      }
      
      close(TMPFILE);
      
   }
   else {
      
      close(TMPFILE);
      $returncode=&doaction(*oldentry,$oldtype,$oldaction,$source,$oldi) if ((!$returncode) && ($i!=$first));
   
   }
   
   $from=$last+1;
   
   return $returncode;
}


#
# Main program

#
#
# check if this source *cannot* be updated with other methods...

if ($CANUPD{$opt_s}) {
   print STDERR "You cannot allow updates to database: $opt_s\n";
   print STDERR "*and* mirror this database at the same time.\n\n";
   print STDERR "Please remove your CANUPD variable for this database in the \'config\' file.\n";

   exit 1;
}

#
# create lock

# print STDERR "files: $locklink and $tmpdata -$$-\n";

if (!symlink($tmpdata,$locklink)) {
   
   local($linkdata)=readlink($locklink);
   
   local($pid)=$linkdata=~ /\.(\d+)$/;
   
   print STDERR "***ERROR***: found other $PROGRAMNAME (PID=$pid) running\n";
   
   print STDERR "please check for presence of process: $pid\n";
   print STDERR "link:  \'$locklink\'\nand file: \'$linkdata\'\n\n";
   print STDERR "The query interval in your cronjob might be too small if this\n";
   print STDERR "problem happens more often and you cannot find these files\n\n";
   print STDERR "If you find these files:\n";
   print STDERR "Remove them to restart automatic mirroring\n\n";
   print STDERR "Please send a bug report to <ripe-dbm\@ripe.net> with\n";
   print STDERR "all data (\'ls -al $LOCKDIR\' and contents of \'$linkdata\')\n";
   print STDERR "when you cannot find an explanation for this problem\n";
   
   exit 1;
}

#
# and now really start running...

local($returncode)=0;
local($period)=0;
local($failedconnects)=0;

local($from,$to,$version,$line,$delay,$outputfound,$msg,$mtime,$oldmtime); 

if ($opt_u) {
   
   ($period,$delay)=split(/\:/, $opt_u);
   
   $SIG{"ALRM"}='timeouthandler';
   
   alarm $period;
   
   #
   # sleep for a random 0..59 seconds to divert
   # the load from all clients on the server even if cron starts
   # them all at the same time
   
   { 
     srand;
     my $random = abs rand(60);

     sleep $random;

   }
   
}

#
# we want to exit as cleanly as possible when we get a signal to do so
   
$SIG{'HUP'}='stopupdating';
$SIG{'INT'}='stopupdating';
$SIG{'KILL'}='stopupdating';
$SIG{'TERM'}='stopupdating';
   

$from=&getcurrentserial($opt_s)+1;

$to="LAST";

# loop until told to stop (signal handlers set global variables)
#  query the server
#  process it
#  sleep (in continue block)
#
QUERY: while ((!$TIMEOUT) && (!$STOPUPDATING)) {

  #
  # do the whois query
   
  if (($msg=&initwhoisqry($::WHOIS_HOST,
			  $::WHOIS_PORT,
			  "-g $opt_s:$UPDATEVERSION:$from-$to" ))) {

    # exit if only doing one query
    &exitsyncdbase($msg) if (!$period);
      
    $failedconnects++;
      
    &exitsyncdbase($msg." ($failedconnects times!)")
      if ($failedconnects*$delay > $period/20);      
   
   }
   
   #
   # slurp the complete input in order to disconnect as soon as possible.

   open(TMPFILE,"+>".$tmpdata) || &fatalerror("cannot open whois output file");

   while ($line=<WHOIS_S>) {
      print TMPFILE $line;
   }

   seek(TMPFILE,0,0);

   $outputfound=0;

   #
   # find output
   
LINE:   
   while (($line=<TMPFILE>) && ($line!~ /^\%START/)) { 
   
      ## If there were no updates available we get '% Warning (1)
      ## Just close the TMPFILE and do the next query
      ## The sleep is in a 'continue' block

      if ($line =~ /\s*%\s+Warning\s+\(1\)/) {
	close TMPFILE;
	&exitsyncdbase("No more updates available") if (!$period);
	next QUERY;
      }

      if (($line=~ /^%/) && ($line=~ /$PROBLEMS/i)) {
   
         close(TMPFILE);
      
         if ($line=~ /ERROR/i) {
            &exitsyncdbase("Error found: ".$line);
            
         }
         else {
            print STDERR "Warning (from raw input data): $line\n"; 
         }
      
      }
   
      next if (($line=~ /^\s*$/) || ($line=~ /^\s*[\*\#]/));

      #
      # for backwards compatibility, can go in next release
      
      last if ($line=~ /^ERROR\:\s+Warning\s+\(1\)/);
      
      $outputfound=1;

   }

   #
   # quit if we found meaningless data, but keep going if we
   # found nothing (usually server timeouts, bad connectivity)
   
   if ($line!~ /^\%START/) {
   
      if ($outputfound) {
         &exitsyncdbase("No valid data found in whois data: $tmpdata");
      }
      else {
         &exitsyncdbase(0) if (!$period);
      }
   
   }
   else {
   
      #
      # call the right updating procedure
   
      if ($line=~ /^\%START\s+Version:\s*(\d+)\s+/) {

         $version=$1;
   
         if ($version==1) {
            ($returncode)=&UpdateVersionOne(TMPFILE,$line,$opt_s,*from, $to, $period);
         
         }
         else {
            &exitsyncdbase("got unsupported update version format ($version), we support versions: ".join(" ",@UPDATEVERSIONS));   
            close(TMPFILE);
         }

         &exitsyncdbase($returncode) if ($returncode);

      }
   
   }
   
   # don't loop round again if a period wasn't specified with $opt_u
   last if (!$period);
   

} continue {

  ## wait before doing another query

  # frequenly stat the master server's *.CURRENTSERIAL file 
  # so we know when there is a new update

  if ($SYNCFASTMODE) {
   
    $oldmtime=(stat($SYNCFASTMODE))[9];
    $mtime=$oldmtime;
      
    while (!$TIMEOUT && $oldmtime==$mtime) {
      sleep $delay;
      $mtime=(stat($SYNCFASTMODE))[9];
    }
   
  }
  else {
    # just sleep for given time

    sleep $delay;
  }
}


&exitsyncdbase($returncode);

# end of program
