#! /usr/bin/perl -w
$ID = q$Id: afsdb-load,v 1.20 2007-04-21 04:50:32 eagle Exp $;
#
# afsdb-load -- Load AFS database from vos listvol output.
#
# Orignally written by Neil Crellin <neilc@stanford.edu>
# Rewritten by Susan Feng <sfeng@stanford.edu>
# Rewritten by Russ Allbery <rra@stanford.edu>
#          and Jonathan Pilat <jonpilat@stanford.edu>
# Copyright 1998, 1999, 2001, 2002, 2005
#     Board of Trustees, Leland Stanford Jr. University
#
# This program is free software; you may redistribute it and/or modify it
# under the same terms as Perl itself.
#
# For each AFS server in the active list, get a list of its partitions and
# then run vos listvol -long on each partition.  Munge the results into a bulk
# load file for Oracle and load it into the volumes table, moving the previous
# contents of the volumes table into previous_volumes so that it can be used
# for nightly reports.

##############################################################################
# Site configuration
##############################################################################

# Where alert mail should be sent if an AFS server cannot be contacted.
$ALERT = 'afs-app' . '@monitoring.stanford.edu';

# Where the mountpoint database is located.
$MOUNTDB = '/afs/ir/service/afs/data/mountpoints/mountpoints.db';

# Number of times to retry vos commands if they fail.
$RETRIES = 10;

# Scratch space to use for temporary files.
$SCRATCH = '/tmp';

# File containing the list of active AFS servers.
$SERVERS = '/afs/ir/service/afs/data/serverlist.afs';

# Find a usable version of vos.
($VOS) = grep { -x $_ } qw(/usr/bin/vos /usr/pubsw/bin/vos);
$VOS ||= '/usr/pubsw/bin/vos';

##############################################################################
# Modules and declarations
##############################################################################

require 5.005;

use strict;
use vars qw($ALERT $ID $MOUNTDB $RETRIES $SCRATCH $SERVERS $VERBOSE $VOS);

use Date::Parse qw(str2time);
use DB_File;
use Getopt::Long qw(GetOptions);
use POSIX qw(strftime);
use Stanford::LSDB::AFSDB;

##############################################################################
# Status reporting
##############################################################################

# Send an alert reporting that an action failed.
sub alert {
    my @action = @_;
    my ($sendmail) = grep { -x $_ } qw(/usr/sbin/sendmail /usr/lib/sendmail);
    $sendmail ||= '/usr/lib/sendmail';
    unless (open (MAIL, "| $sendmail -t -oi -oem")) {
        warn "$0: unable to send alert mail\n";
        warn "$0: attempting to report @action\n";
        return;
    }
    my $date = strftime ('%Y-%m-%d %T' , localtime time);
    print MAIL "To: $ALERT\n";
    print MAIL "Subject: $0 failure ($date)\n";
    print MAIL "\n";
    print MAIL "$0 failed at $date.  Error message:\n\n";
    print MAIL "    @action\n";
    close MAIL;
    if ($? != 0) {
        warn "$0: unable to send alert mail\n";
        warn "$0: attempting to report @action\n";
        return;
    }
}

# If verbosity was requested, say what we're doing with a timestamp.
sub report_action {
    my @action = @_;
    print strftime ('%Y-%m-%d %T ' , localtime time), @_, "\n" if $VERBOSE;
}

# Report a failure of sqlldr, appending the sqlldr log file to the program
# output.  Takes the path to the log file.
sub report_sqlldr_failure {
    my $file = shift;
    warn "Failed to load volumes data.  sqlldr log file follows.\n\n";
    open (LOG, $file) or die "Cannot open $file: $!\n";
    local $_;
    warn $_ while <LOG>;
    close LOG;
    exit 1;
}

##############################################################################
# AFS data munging
##############################################################################

# Given a file containing vos listvol -long output, munge it into a file
# suitable for Oracle sqlldr processing.
sub munge_listvol {
    my $listvol = shift;
    my $mungefile = "$SCRATCH/afsdb-munge.dat";

    # Open the source and output files.
    open (LISTVOL, $listvol) or die "Cannot open $listvol: $!\n";
    open (MUNGE, ">> $mungefile") or die "Cannot open $mungefile: $!\n";
    report_action "Munging $listvol";

    # Read data a paragraph at a time and parse it for the data items that we
    # care about.
    local $/ = '';
    local $_;
    while (<LISTVOL>) {
        s/^Total number of volumes on server \S+ partition \S+: \d+\s*//;
        my ($volname, $volid, $type, $used, $status, $server, $part, $rwid,
            $roid, $bkid, $quota, $created, $updated, $accesses) =
                m(
                  ^(\S+) \s+ (\d+) \s+ (RW|RO|BK) \s+ (\d+)\ K \s+ (\S+) \s+
                  ([^\s.]+)\.(?:(?i)stanford\.edu) \s+ (/vicep[a-z]+) \s+
                  RWrite \s+(\d+)\s+ ROnly \s+(\d+)\s+ Backup \s+(\d+)\s+
                  MaxQuota \s+ (\d+)\ K \s+
                  Creation \s+ ([A-Za-z0-9: ]+) \s+
                  (?:Copy \s+ [A-Za-z0-9: ]+ \s+)?
                  (?:Backup \s+ [A-Za-z0-9: ]+ \s+)?
                  Last\ Update \s+ ([A-Za-z0-9: ]+) \s+
                  (?:(\d+) \s+ accesses\ in\ the\ past\ day)?
                 )x;
        next unless defined $updated;
        $updated = '1970-01-01 00:00:00' if $updated eq 'Never';
        $accesses = 0 unless defined $accesses;
        $created = strftime ('%m/%d/%Y %T', localtime str2time ($created));
        $updated = strftime ('%m/%d/%Y %T', localtime str2time ($updated));
        printf MUNGE "%s;%d;%s;%d;%s;%s;%s;%d;%d;%d;%d;%s;%s;%d\n",
            $volname, $volid, $type, $used, $status, $server, $part, $rwid,
            $roid, $bkid, $quota, $created, $updated, $accesses;
    }
    close LISTVOL;
    if (close MUNGE) {
        return 1;
    } else {
        warn "Unable to write to $mungefile\n";
        return;
    }
}

##############################################################################
# Mountpoint data munging
##############################################################################

# Dump the mountpoint database into a flat semicolon-separated text file.
sub dump_mountpoints {
    my %mtpts;
    my $mungefile = "$SCRATCH/mountpoints.dat";
    report_action 'Dumping mountpoints database';
    tie (%mtpts, 'DB_File', $MOUNTDB)
        or die "Cannot tie $MOUNTDB: $!\n";
    open (OUT, "> $mungefile") or die "Cannot create $mungefile: $!\n";
    my ($volume, $data);
    while (($volume, $data) = each %mtpts) {
        my @data = split ';', $data;
        pop @data;
        print OUT join (';', $volume, @data), "\n";
    }
    close OUT;
    untie %mtpts;
}

##############################################################################
# AFS queries
##############################################################################

# Given an AFS server, return a list of its partitions.
sub afssvr_partlist {
    my $afssvr = shift;
    my $success = 0;
    my $partitions;
    for (my $try = 0; !$success && $try < $RETRIES; $try++) {
        $partitions = `$VOS listpart $afssvr 2>&1`;
        $success = ($? == 0);
    }
    if ($? != 0) {
        alert "$VOS listpart $afssvr exited with status ", ($? >> 8), "\n";
        return;
    }
    my @partitions = grep { m%^/vicep% } split (' ', $partitions);
    return @partitions;
}

# Given an AFS server, run vos listvol -long on each partition on that server,
# saving the results into $LISTVOL/$afssvr/longlistvol.all.
sub afssvr_listvol {
    my $afssvr = shift;
    my @partitions = afssvr_partlist ($afssvr);

    # Now run vos listvol for each partition.
    for my $partition (@partitions) {
        my $tmpfile = "$SCRATCH/afsdb-listvol";
        my $success = 0;
        for (my $try = 0; !$success && $try < $RETRIES; $try++) {
            if ($try > 0) {
                report_action "Retry listvol $afssvr $partition";
            } else {
                report_action "Doing listvol $afssvr $partition";
            }

            # Run vos listvol and filter out unwanted lines.
            my $short_partition = $partition;
            $short_partition =~ s%^/vicep%%;
            open (TMP, "> $tmpfile") or die "Cannot open $tmpfile: $!\n";
            open (VOS, "$VOS listvol $afssvr $partition -long |")
                or die "Cannot fork $VOS listvol $afssvr $partition: $!\n";
            local $_;
            my $lastline;
            while (<VOS>) {
                $lastline = $_ unless /^\s+$/;

                # Save each line to TMP, being careful to abort if we have a
                # write error.
                last unless (print TMP $_);
            }
            close TMP or last;
            close VOS;

            # The vos listvol output is complete if vos exited with zero
            # status and the last line contained a total.
            $success = 1 if ($? == 0 && $lastline =~ /^Total volumes onLine/);
        }

        # Only append the output file to the long output file if the vos
        # command succeeded.
        if ($success) {
            if (munge_listvol ($tmpfile)) {
                unlink $tmpfile;
            } else {
                warn "Munging of $tmpfile for $afssvr $partition failed\n";
            }
        } else {
            alert "vos listvol of $afssvr $partition failed\n";
            close TMP;
        }
    }
}

# For each server in the active server list, call afssvr_listvol to refresh
# the vos listvol output.  Takes the server list from $SERVERS unless a list
# of servers are passed to listvol.
sub listvol {
    my @servers = @_;
    unless (@servers) {
        open (SERVERS, $SERVERS) or die "Cannot open $SERVERS: $!\n";
        @servers = <SERVERS>;
        close SERVERS;
        chomp @servers;
    }

    # Remove munge data file before starting just in case we have detritus
    # left over from a previous run.
    unlink "$SCRATCH/afsdb-munge.dat";
    for (@servers) { afssvr_listvol $_ }
}

##############################################################################
# Database manipulation
##############################################################################

# Move the records from volumes to previous_volumes and truncate volumes.
sub move_volumes_table {
    my $dbh = Stanford::LSDB::AFSDB->connect
        or die "Cannot connect to the AFS database\n";
    report_action 'Truncating previous_volumes';
    $dbh->do ('truncate table previous_volumes');
    report_action 'Copying into previous_volumes';
    $dbh->do ('insert into previous_volumes select * from volumes');
    report_action 'Truncating volumes';
    $dbh->do ('truncate table volumes');
    $dbh->disconnect;
}

# Load the mountpoints table from the output file using sqlldr.
sub load_mountpoints {
    my $control = '/db/afsdb/control/load-mountpoints.ctl';
    report_action 'Loading into mountpoints';
    Stanford::LSDB::AFSDB->sqlldr ($control, "$SCRATCH/mountpoints.log") == 0
          or report_sqlldr_failure ("$SCRATCH/mountpoints.log");
    unlink "$SCRATCH/mountpoints.log";
}

# Load the volumes table from the munged output file using sqlldr.
sub load_volumes_table {
    my $control = '/db/afsdb/control/load-afsdb.ctl';
    report_action 'Loading into volumes';
    Stanford::LSDB::AFSDB->sqlldr ($control, "$SCRATCH/afsdb-load.log") == 0
        or report_sqlldr_failure ("$SCRATCH/afsdb-load.log");
    unlink "$SCRATCH/afsdb-load.log";
}

##############################################################################
# Main routine
##############################################################################

# Read in command-line options.
my ($help, $version);
Getopt::Long::config ('no_ignore_case', 'bundling');
GetOptions ('help|h'    => \$help,
            'verbose|V' => \$VERBOSE,
            'version|v' => \$version) or exit 1;
if ($help) {
    print "Feeding myself to perldoc, please wait....\n";
    exec ('perldoc', '-t', $0);
} elsif ($version) {
    my $version = join (' ', (split (' ', $ID))[1..3]);
    $version =~ s/,v\b//;
    $version =~ s/(\S+)$/($1)/;
    $version =~ tr%/%-%;
    print $version, "\n";
    exit;
}

# Switch to /tmp in case Oracle decides to create a sqlnet.log file.
chdir '/tmp' or die "Cannot cd to /tmp: $!\n";

# Generate our data file, move the old table aside in the database, and load
# the data.
listvol (@ARGV);
move_volumes_table;
load_volumes_table;
dump_mountpoints;
load_mountpoints;
report_action 'All finished';
exit 0;

##############################################################################
# Documentation
##############################################################################

=head1 NAME

afsdb-load - Load AFS volumes data into the AFS database

=head1 SYNOPSIS

afsdb-load [B<-hvV>] [I<server> ...]

=head1 DESCRIPTION

B<afsdb-load> loads AFS volumes data into the volumes table of the AFS
database, moving the previous day's data into previous_volumes for use with
nightly reports.  The data loaded is taken from the output of C<vos listvol
-long>.  The list of active AFS servers is taken from
F</afs/ir/service/afs/data/serverlist.afs>.

A list of servers can be provided on the command line, but this should only
be used for testing (since it will result in an incomplete volumes table in
the database).

Unless the B<-V> flag is given, this script will produce no output unless it
fails, making it suitable to be run by cron.

=head1 OPTIONS

=over 4

=item B<-h>, B<--help>

Print out this documentation (which is done simply by feeding the script to
C<perldoc -t>).

=item B<-v>, B<--version>

Print the version of B<afsdb-load> and exit.

=item B<-V>, B<--verbose>

Print out each major action of the script as it's performed, useful for
debugging.

=back

=head1 FILES

=over 4

=item F</afs/ir/service/afs/data/serverlist.afs>

The list of active AFS servers is taken from this file.  To avoid errors
from attempting to run C<vos listvol -long>, only currently running AFS
servers should be included in this file.

=item F</db/afsdb/control/load-afsdb.ctl>

The B<sqlldr> control file used to load the volume data into the AFS
database.

=item F</tmp/afsdb-listvol>

A temporary output file holding the results of the last C<vos listvol>
command.  This file will be created and removed as each server partition is
processed.

=item F</tmp/afsdb-load.log>

The log file used by B<sqlldr>.  If B<sqlldr> fails, the contents of this
log file will also be included in the output of this script.

=item F</tmp/afsdb-munge.dat>

Where the data to be loaded into the database is put.  This file contains
one line for each volume with fields separated by semicolons and is used as
input to Oracle B<sqlldr>.

=item F</tmp/afsdb.bad>

If loading the data fails, this file will contain the first data line that
B<sqlldr> didn't like.

=back

=head1 AUTHORS

Orignally written by Neil Crellin <neilc@stanford.edu>.  Rewritten by Susan
Feng <sfeng@stanford.edu> to handle temporary failures of C<vos listvol>.
Rewritten by Russ Allbery <rra@stanford.edu> and Jonathan Pilat
<jonpilat@stanford.edu> to use Oracle and the Stanford::LSDB::AFSDB module,
to no longer use the C<-extended> flag of C<vos listvol>, and to be somewhat
more parsimonious about its disk space consumption.

=head1 COPYRIGHT AND LICENSE

Copyright 1998, 1999, 2001, 2002, 2005 Board of Trustees, Leland Stanford
Jr. University.

This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

=head1 SEE ALSO

vos(1), vos_listvol(1)

The current version of this program is available from the AFS reporting
database software page at L<http://www.eyrie.org/~eagle/software/afsdb/>.

=cut