use File::stat;
use File::Basename;
use lib qw(../../eg);

sub refresh_db {
    my($SDB, $CONF) = @_;
    
    if($SDB->{LastRefresh} + $CONF->{RefreshPeriod} < time) {
	for(keys %$SDB) {
	    next if /\:/;
	    unless(-e $CONF->{FileRoot}."/$_") {
		my $delete = $CONF->{FileRoot}."/$_";
		$Response->Debug("deleting file $delete");
		delete $SDB->{$delete};
	    }
	}
	$SDB->{LastRefresh} = time;
	
	recurse( sub {
		     if(! /$CONF->{FileMatch}/) {
			 $Response->Debug("$_ does not match $CONF->{FileMatch}");		     
		     } elsif(-d $_) {
			 $Response->Debug("$_ is a directory");
		     } elsif(-e $_) {
			 $Response->Debug("indexing $_");
			 &index_page($SDB, $_);
		     } else {
			 $Response->Debug("no file for $_");
		     }
		 }
		 , $CONF->{FileRoot}
	       );
    }
}

sub index_page {
    my($SDB, $file) = @_;
    return unless -e $file;

    my $file_data = $SDB->{$file} || '';
    my($mtime) = split(/\:\:/, $file_data, 2);
    $mtime ||= 0;
    if($mtime >= stat($file)->mtime) {
	$Response->Debug("file $file has not been modified recently, last update $mtime");
	return;
    }
    
    if($mtime) {
	for (keys %$SDB) {
	    if(/\:$file/) {
		#$Response->Debug("deleting old key $_");
		delete $SDB->{$_};
	    }
	}
    }
    
    open(FILE, $file) || die("can't read $file: $!");
    my $data = join('', <FILE>);
    close(FILE);

    $data =~ s/\<\%.*?\%\>//sg; # strip ASP code
    $data =~ s/^\#\!.*?\n//s;
    $data =~ s/\<head\>.*?\<title\>\s*(.*?)\s*\</\</is;    
    my $title = $1 || '';
    $title = substr($title, 0, 80);

    $data =~ s/\<[^\>]+\>/ /sg;
    $data =~ s/\&\w+\;//sg;
    $data =~ s/(\,\s|[\s\{\}\(\)%:;=\$\"\'\-\/\#]+)/ /sg;
    while($data =~ s/\s+([A-Z]+)\s+([A-Z]+)\b/ $2 /s) {};
    my $summary = substr($data, 0, 320);

#    $Response->Debug("just parsed $data");
    $data = ' '.$data;

    my @words = split(/\s+/, $title.$data);
    my %words;
    for(@words) {
	next if length($_) < 3;
	$_ =~ s/\W+$//;
	$_ = lc $_;
	next if $words{$_}++;
#	$Response->Debug("indexing word $_:$file");
	$SDB->{$_.':'.$file} = 1;
    }
#    $Response->Debug("indexed words for $file: ".join(", ", sort keys %words));

    $summary =~ s/\:\:\:/\:/gs;
    $title =~ s/\:\:\:/\:/gs;
    $SDB->{$file} = join(':::', stat($file)->mtime, $title, $summary);    
}
