bike/lwvwv
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Removes positional dependency for csv header

Browse Source
This commit is contained in:
Jonathan Rosenbaum 2026-05-04 18:31:27 -04:00
parent 404f0cd56f
commit 1dca98898e
1 changed files with 108 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
google-civic-api.pl
View File

@ -3,24 +3,16 @@ use warnings;
use strict; use strict;
# PORTAL CHANGES - assumes you have roster permission for the State League group # PORTAL CHANGES - assumes you have roster permission for the State League group
# Since the portal came online, you now need to process two different types of cvs files: One for all local Leagues (Local Leagues) & one for the MALs (Roster) # Since the portal came online, CSV columns are accessed by name rather than position
# * * * * * * * * * * # to handle column reordering. The script parses the header row to create a mapping.
# LEAGUE - First Name Preferred First Name Last Name Email Phone Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country League Name League ID Status Original Join Date Expiration Date Unique Contact Id Unique Account Id #
# (diff) (diff) # Required columns (must exist in CSV):
# MAL - First Name Preferred First Name Last Name Phone Email Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country Original Join Date Expiration Date Last Login Date Unique Contact Id Unique Account Id # First Name, Last Name, Email, Phone, Mailing Street, Mailing City,
# Mailing State, Mailing Postal Code, Status
# Required local Leagues MAL #
# Preferred First Name [1] [1] # Optional columns:
# Last Name [2] [2] # League ID (if missing, assumes Members At Large)
# Email [3] [4] # Joined or Original Join Date (for join date)
# Phone [4] [3]
# Mailing Street [5] [5]
# Mailing City [6] [6]
# Mailing State [7] [7]
# Mailing Postal Code [8] [8]
# League ID [11] Assumed
# Status [12] not used not used
# Original Join Date [13] [10]
# Google Civic Information API # Google Civic Information API
# Quota: Queries per minute # Quota: Queries per minute
@ -82,30 +74,57 @@ if ( $ARGV[2] ) {
} }
} }
# Open roster file .. hopefully columns remain the same, # Open roster file
# but @fields can be adjusted as required.
foreach my $file (@files) { foreach my $file (@files) {
#
# $file = $ARGV[1];
my $fh; my $fh;
$| = 1; $| = 1;
open $fh, "<", $file; open $fh, "<", $file or die "Cannot open $file: $!";
my $curl = WWW::Curl::Simple->new(); my $curl = WWW::Curl::Simple->new();
# Read the first line # Read and parse the header line to create column mapping
my $localLeague;
my $header = <$fh>; my $header = <$fh>;
print "Header: $header" unless $emailCsvArg; # Output the header for reference chomp $header;
# Decide how to process print "Header: $header\n" unless $emailCsvArg;
if ( $header !~ /League ID/ ) {
print "Processing file: $file ... No League ID found, must be MAL\n" unless $emailCsvArg; # Parse header to get column indices by name
$localLeague = 0; # This is a Members At Large (MAL) file my @headers = parse_csv_line($header);
my %col; # Column name -> index mapping
for (my $i = 0; $i < @headers; $i++) {
my $h = $headers[$i];
$h =~ s/^"//;
$h =~ s/"$//;
$h =~ s/^\s+|\s+$//g; # Trim whitespace
$col{$h} = $i;
}
# Verify required columns exist
my @required_cols = ('First Name', 'Last Name', 'Email', 'Phone', 'Mailing Street',
'Mailing City', 'Mailing State', 'Mailing Postal Code', 'Status');
my @missing;
foreach my $col_name (@required_cols) {
unless (exists $col{$col_name}) {
push @missing, $col_name;
}
}
if (@missing) {
warn "ERROR: Required columns not found in $file:\n";
warn " Missing: " . join(", ", @missing) . "\n";
warn " Available columns: " . join(", ", @headers) . "\n";
close $fh;
next; # Skip this file
}
# Determine file type based on League ID column presence
my $localLeague;
if (exists $col{'League ID'}) {
print "Processing file: $file ... League ID column found\n" unless $emailCsvArg;
$localLeague = 1;
} }
else { else {
print "Processing file: $file ... League ID found\n" unless $emailCsvArg; print "Processing file: $file ... No League ID column, must be MAL\n" unless $emailCsvArg;
$localLeague = 1; # This is local League file $localLeague = 0;
} }
my $csv = my $csv =
@ -115,41 +134,52 @@ foreach my $file (@files) {
while ( my $line = <$fh> ) { while ( my $line = <$fh> ) {
# my @fields = split( /,/, $line ); my @fields = parse_csv_line($line);
# may be commas between those quotes
# pulled this regexp from https://stackoverflow.com/questions/18144431/regex-to-split-a-csv (solution 45)
my @fields =
$line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g
; #m/("[^"]+"|[^,]+)(?:,\s*)?/g;
# print $#fields . "\n";
my $leagueId, my $phone, my $email, my $joinDate, my $status;
if ($localLeague) {
# Local League file processing # Helper to get field value by column name
# Updated for new CSV header format (as of 2026-05-02) my $get_field = sub {
# First Name[0],Preferred First Name[1],League Name[2],League ID[3],Status[4],Joined[5],Expiration[6],Unique Contact Id[7],Unique Account Id[8],Last Name[9],Email[10],Phone[11],Mailing Street[12],Mailing City[13],Mailing State[14],Mailing Postal Code[15],Mailing Country[16] my ($name) = @_;
( $leagueId = $fields[3] ) =~ s/"//g; # League ID [was 11] my $idx = $col{$name};
( $phone = $fields[11] ) =~ s/"//g; # Phone [was 4] unless (defined $idx) {
( $email = $fields[10] ) =~ s/"//g; # Email [was 3] warn "ERROR: Column '$name' not found in file $file\n";
( $joinDate = $fields[5] ) =~ s/"//g; # Join Date [was 13] warn "Available columns: " . join(", ", keys %col) . "\n";
( $status = $fields[4] ) =~ s/"//g; # Status [was 12] return undef;
}
my $val = $fields[$idx];
$val =~ s/^"//;
$val =~ s/"$//;
return $val;
};
my ($leagueId, $phone, $email, $joinDate, $status);
if ($localLeague) {
$leagueId = $get_field->('League ID');
$phone = $get_field->('Phone');
$email = $get_field->('Email');
$joinDate = $get_field->('Joined');
$status = $get_field->('Status');
} }
else { else {
# Members At Large (MAL) file processing $leagueId = $stateLeagueID; # Default for MALs
# Updated for new CSV header format (as of 2026-05-02) $phone = $get_field->('Phone');
# League ID is assumed to be WV000 for all MALs $email = $get_field->('Email');
$leagueId = $stateLeagueID; # Default for MALs $joinDate = $get_field->('Original Join Date');
( $phone = $fields[11] ) =~ s/"//g; # Phone [was 3] $status = $get_field->('Status');
( $email = $fields[10] ) =~ s/"//g; # Email [was 4] }
( $joinDate = $fields[5] ) =~ s/"//g; # Join Date [was 10]
( $status = $fields[4] ) =~ s/"//g; # Status [was 13] my $firstName = $get_field->('First Name');
my $lastName = $get_field->('Last Name');
my $street = $get_field->('Mailing Street');
my $city = $get_field->('Mailing City');
my $state = $get_field->('Mailing State');
my $zip = $get_field->('Mailing Postal Code');
# Check for undefined values (missing columns)
unless (defined $firstName && defined $lastName) {
warn "ERROR: Missing required First Name or Last Name in record, skipping\n";
next;
} }
( my $firstName = $fields[0] ) =~ s/"//g; # First Name [unchanged]
( my $lastName = $fields[9] ) =~ s/"//g; # Last Name [was 2]
( my $street = $fields[12] ) =~ s/"//g; # Street [was 5]
( my $city = $fields[13] ) =~ s/"//g; # City [was 6]
( my $state = $fields[14] ) =~ s/"//g; # State [was 7]
( my $zip = $fields[15] ) =~ s/"//g; # Zip [was 8]
next if $street eq "Mailing Street"; next if $street eq "Mailing Street";
# Filter for Primary status only (Primary, Primary - Life, etc.) # Filter for Primary status only (Primary, Primary - Life, etc.)
@ -304,6 +334,21 @@ foreach my $file (@files) {
} }
# Helper function to parse a CSV line into fields
# Handles quoted fields and commas within quotes
sub parse_csv_line {
my ($line) = @_;
chomp $line;
# Use the same regex pattern that was already in the code
my @fields = $line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g;
# Remove the extra empty field that the regex sometimes captures at the end
pop @fields while @fields && $fields[-1] eq '';
return @fields;
}
# skip over invalid and nonexistent email addresses # skip over invalid and nonexistent email addresses
sub email { sub email {