—

              
#!/perl
#ABSTRACT: join all csv files in folder into one and remove duplicates
package csvjoin;
$csvjoin::VERSION = '1.01';
# TO-DO: move business logic to CSV::Processor or separate module and write tests
use strict;
use warnings;
# use Regexp::Common;
use Getopt::Long qw(:config auto_help);
use feature 'say';
use Carp;
use Cwd;
use Pod::Usage;
use Text::CSV;
use File::Slurp;
use List::Util qw(uniq);
use CSV::Processor;
$SIG{__DIE__} = sub {
    Carp::longmess( $_[0] );
};
my %opts;
GetOptions(
    "d|dir=s"   => \$opts{dir},
    "i|in=s"    => \$opts{unique_column},
    "j|join=s"  => \$opts{join_column},
    "f|file=s"  => \$opts{out_file},
    "v|verbose" => \$opts{verbose}
) or die("Error in command line arguments\n");
# pod2usage("No obligatory option --in specified\n") unless defined $opts{file};
# say "Verbose mode enabled" if $opts{verbose};
# defaults
$opts{dir} = getcwd unless defined $opts{dir};
say "Reading all csv files in dir " . $opts{dir} if $opts{verbose};
opendir my $dir, $opts{dir} or die "Cannot open directory: $opts{dir} : $!";
my @files = grep { $_ =~ m/\.csv$/ } readdir $dir;
no warnings 'utf8';
say "Found files: " . join( ', ', @files );
use warnings 'utf8';
closedir $dir;
$opts{unique_column} = 'ID' unless defined $opts{unique_column};
say "Looking for unique data in " . $opts{unique_column} if $opts{verbose};
$opts{join_column} = 'SOURCE'     unless defined $opts{join_column};
$opts{out_file}    = 'unique.csv' unless defined $opts{out_file};
say "Unique results will be written in "
  . $opts{out_file}
  . ", joined column is "
  . $opts{join_column}
  if $opts{verbose};
my @new_files;
for my $f (@files) {
    my $bot = CSV::Processor->new( in_file => $f, prefix => 'csvuniq_' );
    $bot->add_same( $opts{unique_column}, $opts{join_column}, value => $f )
      ;    # add source = filename column to each file
    push @new_files, 'csvuniq_' . $f;
}
# `touch $opts{out_file}`;
my $buffer_file = 'joined.csv';
# Combine to single file considering first string as header
for my $i ( 0 .. $#new_files ) {
    my $lines_ref = read_file( $new_files[$i], array_ref => 1 );
    my $header = shift @$lines_ref;
    append_file( $buffer_file, [$header] ) if ( $i eq 0 );
    append_file( $buffer_file, @$lines_ref );
    unlink $new_files[$i];
}
my $csv = Text::AutoCSV->new( in_file => $buffer_file );
$csv->read();
my $nb_rows = $csv->get_nb_rows();
my @cols    = $csv->get_fields_names();
my @ids     = $csv->get_values( $opts{unique_column} );
@ids = uniq @ids;
# say "Found unique ids:\n".join(  "\n", @ids ) if $opts{verbose};
my @data_in_new_csv;
my $repeated_records = 0;
for my $id (@ids) {
    my $found_ar = $csv->search( $opts{unique_column}, $id );
    my $row_hr = $csv->get_row_hr( $found_ar->[0] );
    if ( scalar @$found_ar > 1 ) {
        say "Duplicate found : " . $opts{unique_column} . " : " . $id
          if $opts{verbose};
        $repeated_records++;
        my @a;
        for my $row_number (@$found_ar) {
            push @a, $csv->get_cell( $row_number, $opts{join_column} );
        }
        $row_hr->{ $opts{join_column} } = join( ',', @a )
    }
    push @data_in_new_csv, $row_hr;
}
unlink $buffer_file;
$csv = Text::CSV->new() or die "Cannot use CSV: " . Text::CSV->error_diag();
$csv->eol("\012");
$csv->sep_char(";");
open my $fh, ">:encoding(utf8)", $opts{out_file} or die "$opts{out_file}: $!";
$csv->column_names(@cols);
$csv->print_hr( $fh, $_ ) for @data_in_new_csv;
close $fh or die "$opts{out_file}: $!";
say "Total records in : " . $nb_rows                 if $opts{verbose};
say "Total records out : " . scalar @data_in_new_csv if $opts{verbose};
say "Repeated records : " . $repeated_records        if $opts{verbose};
my $duplicates = $nb_rows - scalar @data_in_new_csv;
say "Total duplicates : " . $duplicates if $opts{verbose};
my $percentage = ( $duplicates / $nb_rows ) * 100;
printf( "CSV reduce percentage : %.2f \n", $percentage ) if $opts{verbose};
# Text::AutoCSV: error: illegal call while read is in progress, would lead to infinite recursion
# $csv->set_walker_hr(sub {
#     my $hr = shift;
#     my $found_ar = $csv->search( $opts{unique_column}, $hr->{ $opts{unique_column} } );
#     my $row_hr = $csv->get_row_hr( $found_ar->[0] );
#
#     if ( scalar @$found_ar > 1 ) {
#         my @a;
#         for my $row_number ( @$found_ar ) {
#             push @a, $csv->get_cell($row_number, $opts{unique_column});
#         }
#         $row_hr->{ $opts{join_column} } = join ( ',' , @a)
#     }
#
#     return $hr;
# });
exit 0;
__END__
HideShow 69 lines of Pod
=pod
=encoding UTF-8
=head1 NAME
csvjoin - join all csv files in folder into one and remove duplicates
=head1 VERSION
version 1.01
=head1 SYNOPSIS
    csvjoin -d </user/pavel/csv> -i <in_column_name_or_index> -o <column name to store source> -v
<in_column_name_or_index> could be column name if csv has header or just row number.
In case of using row number please make sure that csv column structure is same
Read csv files without subdirectories
All available options:
    -d | --dir      name of directory to process. by default is cwd
    -i | --in       number or name of column, data from which will be checked for unique, by default is id or first column
    -j | --join     number or name of column where data is different and where it is needed to join values 
    -f | --file     name of output file, by default is unique.csv
    -v | --verbose  verbose mode
Examples of usage:
    csvjoin
    csvjoin -i 2
    csvjoin ---in ID -v
=head1 DESCRIPTION
CLI tool that makes one csv with unique data from all csv in current folder
Removes duplicates in csv file assuming that one column could be different ( content in this column will be joined )
Criteria that records are same is same field C<ID>. However you can pass your own column names
NOW WORKS FINE ONLY FOR NAMED CSV FILES AND ON LINUX SYSTEMS
=head1 SIMILAR PACKAGES
See more: 
L<Text::CSV>
L<Text::CSV_XS> 
L<Text::CSV::Simple> 
L<Tie::CSV_File>
L<Text::CSV::Merge>
=head1 AUTHOR
Pavel Serikov <pavelsr@cpan.org>
=head1 COPYRIGHT AND LICENSE
This software is copyright (c) 2018 by Pavel Serikov.
This is free software; you can redistribute it and/or modify it under
the same terms as the Perl 5 programming language system itself.
=cut