vhost/cgi-bin/reports-summary.cgi

#!/usr/bin/perl
use strict;
$|++;

my $VERSION = '3.40';

#----------------------------------------------------------------------------

=head1 NAME

reports-summary.cgi - program to return graphical status of a CPAN distribution

=head1 SYNOPSIS

  perl reports-summary.cgi

=head1 DESCRIPTION

Called in a CGI context, returns the current reporting statistics for a CPAN
distribution, depending upon the POST parameters provided.

=cut

# -------------------------------------
# Library Modules

use OpenThought();
use CGI;
use Config::IniFiles;
use CPAN::Testers::Common::DBUtils;
use Template;

use IO::File;
use Data::Dumper;

# -------------------------------------
# Variables

my $DEBUG = 0;
my $LONG_ALLOWED = 0;

my $VHOST = '/var/www/reports/';
my (%options,%cgiparams,%output,$OT,$cgi,$tt);

my %rules = (
    dist    => qr/^([-\w.]+)$/i,
    author  => qr/^([a-z0-9]+)$/i,
    version => qr/^([-\w.]+)$/i,
    grade   => qr/^([0-4])$/i,
    oncpan  => qr/^([0-2])$/i,
    distmat => qr/^([0-2])$/i,
    perlmat => qr/^([0-2])$/i,
    patches => qr/^([0-2])$/i,
    perlver => qr/^([\w.]+)$/i,
    osname  => qr/^([\w.]+)$/i
);

my $EXCEPTIONS;
my %SYMLINKS;
my %MERGED;

# -------------------------------------
# Program

init_options();
process_dist()      if($cgiparams{dist});
process_author()    if($cgiparams{author});
writer();

# -------------------------------------
# Subroutines

sub init_options {
    $options{config} = $VHOST . 'cgi-bin/config/settings.ini';

    error("Must specific the configuration file\n")             unless($options{config});
    error("Configuration file [$options{config}] not found\n")  unless(-f $options{config});

    # load configuration
    my $cfg = Config::IniFiles->new( -file => $options{config} );

    # configure upload DB
    for my $db (qw(CPANSTATS)) {
        my %opts = map {$_ => $cfg->val($db,lc $db . '_' . $_);} qw(driver database dbfile dbhost dbport dbuser dbpass);
        $options{$db} = CPAN::Testers::Common::DBUtils->new(%opts);
        error("Cannot configure '$options{database}' database\n")   unless($options{$db});
    }

    $cgi = CGI->new;

    $options{format} = $cgi->param('format') || 'html';
    $options{format} = 'html' unless($options{format} =~ /html|txt|xml/);

    #audit("DEBUG: configuration done");

    for my $key (keys %rules) {
        my $val = $cgi->param("${key}_pref") || $cgi->param($key);
        $cgiparams{$key} = $1   if($val =~ $rules{$key});
    }

    #audit('DEBUG: cgiparams=',Dumper(\%cgiparams));

    # set up API to Template Toolkit
    $tt = Template->new(
        {
            #    POST_CHOMP => 1,
            #    PRE_CHOMP => 1,
            #    TRIM => 1,
            EVAL_PERL    => 1,
            INCLUDE_PATH => [ 'templates' ],
        }
    );

    my $config = $VHOST . 'cgi-bin/config/cpan-config.ini';
    error("Must specific the configuration file\n")             unless($config);
    error("Configuration file [$config] not found\n")           unless(-f $config);

    $cfg = Config::IniFiles->new( -file => $config );

    if($cfg->SectionExists('EXCEPTIONS')) {
        my @values = $cfg->val('EXCEPTIONS','LIST');
        $EXCEPTIONS = join('|',@values);
    }

    if($cfg->SectionExists('SYMLINKS')) {
        $SYMLINKS{$_} = $cfg->val('SYMLINKS',$_)  for($cfg->Parameters('SYMLINKS'));
        push @{$MERGED{$SYMLINKS{$_}}}, $_              for(keys %SYMLINKS);
        push @{$MERGED{$SYMLINKS{$_}}}, $SYMLINKS{$_}   for(keys %SYMLINKS);
    }

    #audit('DEBUG: SYMLINKS=',Dumper(\%SYMLINKS));
    #audit('DEBUG: MERGED=',Dumper(\%MERGED));

    #$cgiparams{dist} = 'App-Maisha';
    #$cgiparams{author} = 'BARBIE';
}

sub process_dist {
    $cgiparams{dist} = $SYMLINKS{$cgiparams{dist}}  if(defined $SYMLINKS{$cgiparams{dist}});

    if( $LONG_ALLOWED && 
        ( ($cgiparams{osname}  && $cgiparams{osname}  =~ /ALL/i) ||
          ($cgiparams{perlver} && $cgiparams{perlver} =~ /ALL/i) ) ) {
        process_dist_long();
    } else {
        process_dist_short();
    }
}

sub process_dist_short {
    #audit("DEBUG: start process dist (short): $cgiparams{dist}");

    my @where;
    push @where, "patched=$cgiparams{patches}"      if($cgiparams{patches});
    push @where, "distmat=$cgiparams{distmat}"      if($cgiparams{distmat});
    push @where, "perlmat=$cgiparams{perlmat}"      if($cgiparams{perlmat});
    push @where, "oncpan=$cgiparams{oncpan}"        if($cgiparams{oncpan});
    push @where, "version='$cgiparams{version}'"    if($cgiparams{version});
    my $where = @where ? ' AND ' . join(' AND ',@where) : '';

    my $dist = "'$cgiparams{dist}'";
    $dist = "'" . join("','",@{$MERGED{$cgiparams{dist}}}) . "'"    if(defined $MERGED{$cgiparams{dist}});
    my $sql = "SELECT version,sum(pass) as pass, sum(fail) as fail, sum(na) as na, sum(unknown) as unknown FROM release_summary" .
    	" WHERE dist IN ($dist) $where GROUP BY version ORDER BY version";

    #audit("DEBUG: sql=$sql");

    my $next = $options{CPANSTATS}->iterator('hash', $sql );

    my ( $summary );
    while ( my $row = $next->() ) {
        next unless $row->{version};

        $summary->{ $row->{version} }->{ PASS }    = $row->{pass};
        $summary->{ $row->{version} }->{ FAIL }    = $row->{fail};
        $summary->{ $row->{version} }->{ NA }      = $row->{na};
        $summary->{ $row->{version} }->{ UNKNOWN } = $row->{unknown};
        $summary->{ $row->{version} }->{ ALL }     = $row->{pass} + $row->{fail} + $row->{na} + $row->{unknown};
    }

    #audit("DEBUG: summary=".Dumper($summary));

    my $oncpan = q!'cpan','upload','backpan'!;
    $oncpan = q!'cpan','upload'!    if($cgiparams{oncpan} && $cgiparams{oncpan} == 1);
    $oncpan = q!'backpan'!          if($cgiparams{oncpan} && $cgiparams{oncpan} == 2);

    my @versions;
    if($cgiparams{version}) {
        @versions = ($cgiparams{version});
    } else {
        # ensure we cover all known versions
        my @rows = $options{CPANSTATS}->get_query(
                        'array',
                        "SELECT DISTINCT(version) FROM uploads WHERE dist IN ($dist) AND type IN ($oncpan) ORDER BY released DESC" );
        for(@rows) {
            next    if($cgiparams{distmat} && $cgiparams{distmat} == 1     && $_->[0]  =~ /_/i);
            next    if($cgiparams{distmat} && $cgiparams{distmat} == 2     && $_->[0]  !~ /_/i);
            push @versions, $_->[0];
        }
    }

    my %versions = map {my $v = $_; $v =~ s/[^\w\.\-]/X/g; $_ => $v} @versions;

    #audit("DEBUG: versions=".Dumper(\%versions));

    %output = (
        template  => 'dist_summary',
        variables => {
            versions        => \@versions,
            versions_tag    => \%versions,
            summary         => $summary,
            distro          => $cgiparams{dist}
        }
    );
}

sub process_dist_long {
    #audit("DEBUG: start process dist (long): $cgiparams{dist}");

    my @where;
    push @where, "perl NOT LIKE '%patch%'"          if($cgiparams{patches} && $cgiparams{patches} == 1);
    push @where, "perl LIKE '%patch%'"              if($cgiparams{patches} && $cgiparams{patches} == 2);
    push @where, "version NOT LIKE '%\\_%'"         if($cgiparams{distmat} && $cgiparams{distmat} == 1);
    push @where, "version LIKE '%\\_%'"             if($cgiparams{distmat} && $cgiparams{distmat} == 2);
    push @where, "version='$cgiparams{version}'"    if($cgiparams{version});
    #push @where, "perl NOT REGEX '^5.(7|9|11)'"     if($cgiparams{perlmat} && $cgiparams{perlmat} == 1);
    #push @where, "perl REGEX '^5.(7|9|11)'"         if($cgiparams{perlmat} && $cgiparams{perlmat} == 2);
    my $where = @where ? ' AND ' . join(' AND ',@where) : '';

    my $dist = "'$cgiparams{dist}'";
    $dist = "'" . join("','",@{$MERGED{$cgiparams{dist}}}) . "'"    if(defined $MERGED{$cgiparams{dist}});
    my $sql = "SELECT id, state, version, perl, osname, FROM cpanstats WHERE dist IN ($dist) AND state != 'cpan' $where ORDER BY version, id";

    #audit("DEBUG: sql=$sql");

    my $next = $options{CPANSTATS}->iterator('hash', $sql );

    my ( $summary );
    while ( my $row = $next->() ) {
        next unless $row->{version};
        $row->{perl} = "5.004_05" if $row->{perl} eq "5.4.4"; # RT 15162
        #next    if($cgiparams{patches} && $cgiparams{patches} == 1     && $row->{perl}     =~ /patch/i);
        #next    if($cgiparams{patches} && $cgiparams{patches} == 2     && $row->{perl}     !~ /patch/i);
        #next    if($cgiparams{distmat} && $cgiparams{distmat} == 1     && $row->{version}  =~ /_/i);
        #next    if($cgiparams{distmat} && $cgiparams{distmat} == 2     && $row->{version}  !~ /_/i);
        next    if($cgiparams{perlmat} && $cgiparams{perlmat} == 1     && $row->{perl}     =~ /^5.(7|9|11)/);
        next    if($cgiparams{perlmat} && $cgiparams{perlmat} == 2     && $row->{perl}     !~ /^5.(7|9|11)/);
        next    if($cgiparams{perlver} && $cgiparams{perlver} ne 'ALL' && $row->{perl}     !~ /$cgiparams{perlver}/i);
        next    if($cgiparams{osname}  && $cgiparams{osname}  ne 'ALL' && $row->{osname}   !~ /$cgiparams{osname}/i);

        $summary->{ $row->{version} }->{ uc $row->{state} }++;
        $summary->{ $row->{version} }->{ 'ALL' }++;
    }

    #audit("DEBUG: summary=".Dumper($summary));

    my $oncpan = q!'cpan','upload','backpan'!;
    $oncpan = q!'cpan','upload'!    if($cgiparams{oncpan} && $cgiparams{oncpan} == 1);
    $oncpan = q!'backpan'!          if($cgiparams{oncpan} && $cgiparams{oncpan} == 2);

    # ensure we cover all known versions
    my @rows = $options{CPANSTATS}->get_query(
                    'array',
                    "SELECT DISTINCT(version) FROM uploads WHERE dist IN ($dist) AND type IN ($oncpan) ORDER BY released DESC" );
    my @versions;
    for(@rows) {
        next    if($cgiparams{distmat} && $cgiparams{distmat} == 1     && $_->[0]  =~ /_/i);
        next    if($cgiparams{distmat} && $cgiparams{distmat} == 2     && $_->[0]  !~ /_/i);
        push @versions, $_->[0];
    }
    my %versions = map {my $v = $_; $v =~ s/[^\w\.\-]/X/g; $_ => $v} @versions;

    #audit("DEBUG: versions=".Dumper(\%versions));

    %output = (
        template  => 'dist_summary',
        variables => {
            versions        => \@versions,
            versions_tag    => \%versions,
            summary         => $summary,
            distro          => $cgiparams{dist}
        }
    );
}

sub process_author {
    if( $LONG_ALLOWED && 
        ( ($cgiparams{osname}  && $cgiparams{osname}  =~ /ALL/i) ||
          ($cgiparams{perlver} && $cgiparams{perlver} =~ /ALL/i) ) ) {
        process_author_long();
    } else {
        process_author_short();
    }
}

sub process_author_short {
    #audit("DEBUG: start process author (short): $cgiparams{author}");
    my (@dists,%summary);

    my @where;
    push @where, "s.patched=$cgiparams{patches}"      if($cgiparams{patches});
    push @where, "s.distmat=$cgiparams{distmat}"      if($cgiparams{distmat});
    push @where, "s.perlmat=$cgiparams{perlmat}"      if($cgiparams{perlmat});
    push @where, "s.oncpan=$cgiparams{oncpan}"        if($cgiparams{oncpan});
    my $where = @where ? ' AND ' . join(' AND ',@where) : '';

    my $sql =   q{SELECT x.dist FROM ixlatest AS x WHERE x.author=? AND x.version IS NOT NULL AND x.version!=''};
    my @rows = $options{CPANSTATS}->get_query('hash', $sql, $cgiparams{author} );
    for my $row (@rows) {
        $summary{ $row->{dist} }->{ PASS }    = 0;
        $summary{ $row->{dist} }->{ FAIL }    = 0;
        $summary{ $row->{dist} }->{ NA }      = 0;
        $summary{ $row->{dist} }->{ UNKNOWN } = 0;
        $summary{ $row->{dist} }->{ ALL }     = 0;
    }

    $sql =  'SELECT x.dist,sum(s.pass) as pass,sum(s.fail) as fail,sum(s.na) as na,sum(s.unknown) as unknown ' .
            'FROM ixlatest AS x ' .
            'LEFT JOIN release_summary AS s ON x.dist=s.dist AND x.version=s.version ' .
            "WHERE x.author=? AND s.version IS NOT NULL AND s.version!='' $where " .
            'GROUP BY x.dist';

    #audit("DEBUG: sql=$sql");

    #my $next = $options{CPANSTATS}->iterator('hash', $sql, $cgiparams{author} );
    my @rows = $options{CPANSTATS}->get_query('hash', $sql, $cgiparams{author} );
    #audit("DEBUG: rows=".(scalar(@rows)));

    #while ( my $row = $next->() ) {
    for my $row (@rows) {
        #audit("DEBUG: processing $inx/$max: $row->{dist} [$row->{type}]");

        next    unless($row->{dist} =~ /^[A-Za-z0-9][A-Za-z0-9\-_]*$/
                    || $row->{dist} =~ /$EXCEPTIONS/);

        $summary{ $row->{dist} }->{ PASS }    += $row->{pass};
        $summary{ $row->{dist} }->{ FAIL }    += $row->{fail};
        $summary{ $row->{dist} }->{ NA }      += $row->{na};
        $summary{ $row->{dist} }->{ UNKNOWN } += $row->{unknown};
        $summary{ $row->{dist} }->{ ALL }     += $row->{pass} + $row->{fail} + $row->{na} + $row->{unknown};
    }

    my @dists = map {
            {
                distribution => $_,
                summary      => $summary{$_},
            }
        } sort keys %summary;

    #audit("DEBUG: summary data retrieved");
    #audit("DEBUG: dists=".Dumper(\@dists));

    %output = (
        template  => 'author_summary',
        variables => {
            distributions   => \@dists,
        }
    );
}

sub process_author_long {
    #audit("DEBUG: start process author: $cgiparams{author}");
    my (@dists,%summary);

    my @where;
    push @where, "u.type != 'backpan'"              if($cgiparams{oncpan}  && $cgiparams{oncpan}  == 1);
    push @where, "u.type = 'backpan'"               if($cgiparams{oncpan}  && $cgiparams{oncpan}  == 2);
    push @where, "c.perl NOT LIKE '%patch%'"        if($cgiparams{patches} && $cgiparams{patches} == 1);
    push @where, "c.perl LIKE '%patch%'"            if($cgiparams{patches} && $cgiparams{patches} == 2);
    push @where, "c.version NOT LIKE '%\\_%'"       if($cgiparams{distmat} && $cgiparams{distmat} == 1);
    push @where, "c.version LIKE '%\\_%'"           if($cgiparams{distmat} && $cgiparams{distmat} == 2);
    #push @where, "c.perl NOT REGEXP '^5.(7|9|11)'"  if($cgiparams{perlmat} && $cgiparams{perlmat} == 1);
    #push @where, "c.perl REGEXP '^5.(7|9|11)'"      if($cgiparams{perlmat} && $cgiparams{perlmat} == 2);
    my $where = @where ? ' AND ' . join(' AND ',@where) : '';

    my $sql =   'SELECT c.dist,c.state,c.perl,c.osname FROM cpanstats AS c ' .
                'INNER JOIN ixlatest AS x ON x.dist=c.dist AND x.version=c.version ' .
                'INNER JOIN uploads AS u ON u.dist=x.dist AND u.version=x.version ' .
                "WHERE x.author=? $where ORDER BY id";

    #audit("DEBUG: sql=$sql");

    #my $next = $options{CPANSTATS}->iterator('hash', $sql, $cgiparams{author} );
    my @rows = $options{CPANSTATS}->get_query('hash', $sql, $cgiparams{author} );
    #audit("DEBUG: rows=".(scalar(@rows)));

    my $inx = 0;
    my $max = 0;#scalar(@rows);
    #while ( my $row = $next->() ) {
    for my $row (@rows) {
        $inx++;
        #audit("DEBUG: processing $inx/$max: $row->{dist} [$row->{type}]");

        next    unless($row->{dist} =~ /^[A-Za-z0-9][A-Za-z0-9\-_]*$/
                    || $row->{dist} =~ /$EXCEPTIONS/);

        $row->{perl} = "5.004_05" if $row->{perl} eq "5.4.4"; # RT 15162
        next    if($cgiparams{perlmat} && $cgiparams{perlmat} == 1     && $row->{perl}     =~ /^5.(7|9|11)/);
        next    if($cgiparams{perlmat} && $cgiparams{perlmat} == 2     && $row->{perl}     !~ /^5.(7|9|11)/);
        next    if($cgiparams{perlver} && $cgiparams{perlver} ne 'ALL' && $row->{perl}     !~ /$cgiparams{perlver}/i);
        next    if($cgiparams{osname}  && $cgiparams{osname}  ne 'ALL' && $row->{osname}   !~ /$cgiparams{osname}/i);

        $summary{$row->{dist}}->{ uc $row->{state} }++;
        $summary{$row->{dist}}->{ 'ALL' }++;
    }

    my @dists = map {
            {
                distribution => $_,
                summary      => $summary{$_},
            }
        } sort keys %summary;

    #audit("DEBUG: summary data retrieved");
    #audit("DEBUG: dists=".Dumper(\@dists));

    %output = (
        template  => 'author_summary',
        variables => {
            distributions   => \@dists,
        }
    );
}

sub writer {
    my $result;

    my $template = $output{template} . '.' . $options{format};
    unless(-f "templates/$template") {
        print $cgi->header('text/text','404 Page Not Found');
        print "Invalid data request\n";
        return;
    }

    $tt->process( $template, $output{variables}, \$result )
            || error( $tt->error );

    $result =~ s/\s{2,}/ /g;

    #audit("DEBUG: result=$result");

    if($options{format} eq 'xml') {
        print $cgi->header('text/xml') . $result . "\n";
    } elsif($options{format} eq 'txt') {
        print $cgi->header('text/text') . $result . "\n";
    } else {
        $OT = OpenThought->new();

        my $html;
        $html->{'reportsummary'} = $result;
        $OT->param( $html );

        #audit("DEBUG: response=" . $OT->response());

        print $cgi->header;
        print $OT->response();
    }
}

sub error {
    audit('ERROR:',@_);
    print STDERR @_;
    print $cgi->header('text/plain'), "Error retrieving data\n";
    exit;
}

sub audit {
    return  unless($DEBUG);

    my @date = localtime(time);
    my $date = sprintf "%04d/%02d/%02d %02d:%02d:%02d", $date[5]+1900, $date[4]+1, $date[3], $date[2], $date[1], $date[0];

    my $fh = IO::File->new($VHOST . 'cgi-bin/cache/summary-audit.log','a+') or return;
    print $fh "$date " . join(' ',@_ ). "\n";
    $fh->close;
}

1;

__END__

=head1 BUGS, PATCHES & FIXES

There are no known bugs at the time of this release. However, if you spot a
bug or are experiencing difficulties, that is not explained within the POD
documentation, please send bug reports and patches to the RT Queue (see below).

Fixes are dependant upon their severity and my availablity. Should a fix not
be forthcoming, please feel free to (politely) remind me.

RT: http://rt.cpan.org/Public/Dist/Display.html?Name=CPAN-WWW-Testers

=head1 SEE ALSO

L<CPAN::WWW::Testers::Generator>
L<CPAN::Testers::WWW::Statistics>

F<http://www.cpantesters.org/>,
F<http://stats.cpantesters.org/>

=head1 AUTHOR

  Barbie, <barbie@cpan.org>
  for Miss Barbell Productions <http://www.missbarbell.co.uk>.

=head1 COPYRIGHT AND LICENSE

  Copyright (C) 2008-2012 Barbie <barbie@cpan.org>

  This module is free software; you can redistribute it and/or
  modify it under the Artistic License 2.0.

=cut
	Global
`s`	Focus search bar
`?`	Bring up this help dialog
	GitHub
`g` `p`	Go to pull requests
`g` `i`	Go to GitHub issues (only if GitHub is preferred repository)
	POD
`g` `a`	Go to author
`g` `c`	Go to changes
`g` `i`	Go to issues
`g` `d`	Go to dist
`g` `r`	Go to repository/SCM
`g` `s`	Go to source
`g` `b`	Go to file browse
Search terms
module: (e.g. module:Plugin)
distribution: (e.g. distribution:Dancer auth)
author: (e.g. author:SONGMU Redis)
version: (e.g. version:1.00)
Keyboard Shortcuts
