bike/lwvwv
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updates to new format for https://portal.lwv.org. Now requires two different csv files.

Browse Source
This commit is contained in:
Jonathan Rosenbaum 2025-03-30 15:38:30 -04:00
parent 40fe62fbdd
commit 778d10b7bf
1 changed files with 219 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
google-civic-api.pl
View File

@ -1,6 +1,26 @@
#!/usr/bin/perl -w
use strict;
# PORTAL CHANGES - assumes you have roster permission for the State League group
# Since the portal came online, you now need to process two different types of cvs files: One for all local Leagues (Local Leagues) & one for the MALs (Roster)
# * * * * * * * * * *
# LEAGUE - First Name Preferred First Name Last Name Email Phone Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country League Name League ID Status Original Join Date Expiration Date Unique Contact Id Unique Account Id
# (diff) (diff)
# MAL - First Name Preferred First Name Last Name Phone Email Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country Original Join Date Expiration Date Last Login Date Unique Contact Id Unique Account Id
# Required local Leagues MAL
# Preferred First Name [1] [1]
# Last Name [2] [2]
# Email [3] [4]
# Phone [4] [3]
# Mailing Street [5] [5]
# Mailing City [6] [6]
# Mailing State [7] [7]
# Mailing Postal Code [8] [8]
# League ID [11] Assumed
# Status [12] not used not used
# Original Join Date [13] [10]
# Google Civic Information API
# Quota: Queries per minute
# Current limit: 151
@ -21,6 +41,7 @@ use strict;
# cpanm Module::Name
use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required
use WWW::Curl::Simple;
use File::Glob ':glob';
use JSON;
use Data::Dumper;
use vars qw($key);
@ -33,6 +54,7 @@ my $googleApiKey = $key;
# Allow normal queries, too
my $searchType = $ARGV[0];
my @files;
if ( !$ARGV[0] ) {
help();
exit;
@ -44,6 +66,8 @@ else {
print "WRONG FIRST ARGUMENT\n";
exit;
}
@files = bsd_glob( $ARGV[1] ); # Expand wildcard patterns (e.g., *.csv)
}
my $emailArg;
@ -53,38 +77,72 @@ if ( $ARGV[2] ) {
# Open roster file .. hopefully columns remain the same,
# but @fields can be adjusted as required.
my $file;
$file = $ARGV[1];
my $fh;
$| = 1;
open $fh, "<", $file;
foreach my $file (@files) {
my $curl = WWW::Curl::Simple->new();
#
# $file = $ARGV[1];
my $fh;
$| = 1;
open $fh, "<", $file;
my $csv =
"Name,Email,Phone,Address,Delegate District,Senate District,League ID,Join Date\n";
print $csv;
$csv = "";
while ( my $line = <$fh> ) {
my $curl = WWW::Curl::Simple->new();
# my @fields = split( /,/, $line );
# may be commas between those quotes
# pulled this regexp from https://stackoverflow.com/questions/18144431/regex-to-split-a-csv (solution 45)
my @fields = $line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g; #m/("[^"]+"|[^,]+)(?:,\s*)?/g;
# Read the first line
my $localLeague;
my $header = <$fh>;
print "Header: $header"; # Output the header for reference
# Decide how to process
if ( $header !~ /League ID/ ) {
print "Processing file: $file ... No League ID found, must be MAL\n";
$localLeague = 0; # This is a Members At Large (MAL) file
}
else {
print "Processing file: $file ... League ID found\n";
$localLeague = 1; # This is local League file
}
my $csv =
"Name,Email,Phone,Address,Delegate District,Senate District,League ID,Join Date\n";
print $csv;
$csv = "";
while ( my $line = <$fh> ) {
# my @fields = split( /,/, $line );
# may be commas between those quotes
# pulled this regexp from https://stackoverflow.com/questions/18144431/regex-to-split-a-csv (solution 45)
my @fields =
$line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g
; #m/("[^"]+"|[^,]+)(?:,\s*)?/g;
# print $#fields . "\n";
( my $leagueId = $fields[1] ) =~ s/"//g; # League
( my $firstName = $fields[4] ) =~ s/"//g; # First Name
( my $lastName = $fields[5] ) =~ s/"//g; # Last Name
( my $status = $fields[7] ) =~ s/"//g; # Active
( my $joinDate = $fields[10] ) =~ s/"//g; # Join Date
( my $phone = $fields[11] ) =~ s/"//g; # Phone
( my $email = $fields[12] ) =~ s/"//g; # Email
( my $street = $fields[13] ) =~ s/"//g; # Street
( my $city = $fields[14] ) =~ s/"//g; # City
( my $state = $fields[15] ) =~ s/"//g; # State
( my $zip = $fields[17] ) =~ s/"//g; # Zip
my $leagueId, my $phone, my $email, my $joinDate;
if ($localLeague) {
# Local League file processing
# League ID is in column 11
( $leagueId = $fields[11] ) =~ s/"//g; # League ID
( $phone = $fields[4] ) =~ s/"//g; # Phone
( $email = $fields[3] ) =~ s/"//g; # Email
( $joinDate = $fields[13] ) =~ s/"//g; # Join Date
}
else {
# Members At Large (MAL) file processing
# League ID is assumed to be WV000 for all MALs
$leagueId = "WV000"; # Default for MALs
( $phone = $fields[3] ) =~ s/"//g; # Phone
( $email = $fields[4] ) =~ s/"//g; # Email
( $joinDate = $fields[10] ) =~ s/"//g; # Join Date
}
( my $firstName = $fields[0] ) =~ s/"//g; # First Name
( my $lastName = $fields[2] ) =~ s/"//g; # Last Name
( my $street = $fields[5] ) =~ s/"//g; # Street
( my $city = $fields[6] ) =~ s/"//g; # City
( my $state = $fields[7] ) =~ s/"//g; # State
( my $zip = $fields[8] ) =~ s/"//g; # Zip
my $status = "true";
next if $street eq "Mailing Street";
if ( $leagueId !~ /^WV/ && $leagueId ne "LWV Of West Virginia") {
if ( $leagueId !~ /^WV/ && $leagueId ne "LWV Of West Virginia" ) {
next;
}
@ -213,8 +271,10 @@ while ( my $line = <$fh> ) {
}
}
}
}
close $fh;
}
close $fh;
# skip over invalid and nonexistent email addresses
sub email {
@ -242,9 +302,15 @@ sub help {
Copyright (C) 2023 - by Jonathan Rosenbaum <freesource\@freesoftwarepc.com>
This may be freely redistributed under the terms of the GNU General Public License
Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate District for all LWVWV members)
$0 WV000 ./roster-file (show all information for members at large, but do not query Google)
$0 WV000 ./roster-file email (only show email addresses for members at large, but do not query Google)
Note: Since https://portal.lwvwv.org came online, two csv files are now available, which should both be processed:
1. Local Leagues (Local Leagues) - which has all the local League members
2. MAL (Roster) - which has all the members at large
Depending on your request, this script will process either of those files, and query the Google Civic Information API for the Delegate and Senate Districts,
generate an information file or email file.
Usage: $0 google '*.csv' (queries Google Civic Api Delegate and Senate District for all LWVWV members)
$0 WV000 '*.csv' (show all information for members at large, but do not query Google)
$0 WV000 '*.csv' email (only show email addresses for members at large, but do not query Google)
1st argument can be one of these types:
google (all members with senate/delegate district query) which prints out this csv data:
@ -259,7 +325,7 @@ Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate D
You will want to send results to a file.
Example: $0 google ./roster-file > 2023-districts
Example: $0 google '*.csv' > 2023-districts
\n";
print $help;