diff --git a/google-civic-api.pl b/google-civic-api.pl new file mode 100644 index 0000000..2f46836 --- /dev/null +++ b/google-civic-api.pl @@ -0,0 +1,262 @@ +#!/usr/bin/perl -w +use strict; + +# Google Civic Information API +# Quota: Queries per minute +# Current limit: 151 +# +# Use sleep to limit queries to 30 per minute +# Currently there are 187 members in the whole state + +# commandline options: type roster-file email +# type: +# google (all members with senate/delegate district query) +# ALL (all members without senate/delegate district query) +# League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson) +# email (optional third arg): +# email (emails for specified type ALL (or google) or League ID) + +# cpan App::cpanminus +# Now install any module you can find. +# cpanm Module::Name +use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required +use WWW::Curl::Simple; +use JSON; +use Data::Dumper; +use vars qw($key); +require "./env"; + +# Configuration - key for LWVWV +# put it in a file call env with one line $key = "thekeyfromgoogle" +my $googleApiKey = $key; + +# Allow normal queries, too +my $searchType = $ARGV[0]; + +if ( !$ARGV[0] ) { + help(); + exit; +} +else { + if ( $ARGV[0] !~ /google|ALL|WV000|WV102|WV103|WV107|WV112/ ) { + print "WRONG FIRST ARGUMENT\n"; + help(); + print "WRONG FIRST ARGUMENT\n"; + exit; + } +} + +my $emailArg; +if ( $ARGV[2] ) { + $emailArg = $ARGV[2]; +} + +# Open roster file .. hopefully columns remain the same, +# but @fields can be adjusted as required. +my $file; +$file = $ARGV[1]; +my $fh; +$| = 1; +open $fh, "<", $file; + +my $curl = WWW::Curl::Simple->new(); + +my $csv = + "Name,Email,Phone,Address,Join Date,Delegate District,Senate District,\n"; +print $csv; +$csv = ""; +while ( my $line = <$fh> ) { + + # my @fields = split( /,/, $line ); + # may be commas between those quotes + my @fields = $line =~ m/("[^"]+"|[^,]+)(?:,\s*)?/g; + ( my $leagueId = $fields[0] ) =~ s/"//g; # League + ( my $firstName = $fields[4] ) =~ s/"//g; # First Name + ( my $lastName = $fields[5] ) =~ s/"//g; # Last Name + ( my $status = $fields[6] ) =~ s/"//g; # Active + ( my $joinDate = $fields[9] ) =~ s/"//g; # Join Date + ( my $phone = $fields[10] ) =~ s/"//g; # Phone + ( my $email = $fields[11] ) =~ s/"//g; # Email + ( my $street = $fields[12] ) =~ s/"//g; # Street + ( my $city = $fields[13] ) =~ s/"//g; # City + ( my $state = $fields[14] ) =~ s/"//g; # State + ( my $zip = $fields[16] ) =~ s/"//g; # Zip + next if $street eq "Mailing Street"; + next if $leagueId !~ /^WV/; + + if ( $searchType ne "google" + && $searchType ne "ALL" ) + { + next if $leagueId ne $searchType; + } + + # no need for commas now + $street =~ s/,//g; # Street + + if ( $status ne "Inactive" ) { + + if ( !$emailArg ) { + + # contact for people with email address + if ( email($email) ) { + $csv = "$firstName $lastName,"; + $csv .= "$email,"; + } + else { + $csv = "$firstName $lastName,"; + $csv .= ","; + } + + if ($phone) { + + #print "PHONE: $fields[10]\n"; + $csv .= "$phone,"; + } + else { + #print "PHONE: none\n"; + $csv .= ","; + } + + #print "ADDRESS $fields[12] $fields[13] $fields[14] $fields[16]\n"; + + # Check for actual street + if ($street) { + + # Correct known incorrect addresses + if ( $street eq "309 2nd St Altizer" ) { + $street =~ s/309 2nd St Altizer/309 2nd St/; + } + if ( $street eq "!87 Gallaher Street" ) { + $street =~ s/\!87 Gallaher Street/187 Gallaher Street/; + } + + $csv .= "$street $city $state $zip,"; + + # url encoding + $street =~ s/#/%23/; + + # Address query + my $address = "$street $city $state $zip\n"; + + if ( $searchType eq "google" ) { + + # Delegate District Query + my $queryDelegateDistrict = +"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorLowerBody&" + . "key=${googleApiKey}&" + . "address=${address}"; + + sleep 2; + my $res = $curl->get($queryDelegateDistrict); + my $content = decode_json( $res->content ); + + #print Dumper($content); + + my $divisionNumber = + ( ( keys %{ $content->{divisions} } )[0] ); + if ( $divisionNumber && $state eq "WV" ) { + my @divisionNumber = split /:/, $divisionNumber; + + #print $divisionNumber[3] . "\n"; + $csv .= "$divisionNumber[3],"; + } + else { + $csv .= ","; + } + + # Senate District Query + my $querySenateDistrict = +"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorUpperBody&" + . "key=${googleApiKey}&" + . "address=${address}"; + + sleep 2; + $res = $curl->get($querySenateDistrict); + $content = decode_json( $res->content ); + + # print Dumper($content); + $divisionNumber = + ( ( keys %{ $content->{divisions} } )[0] ); + if ( $divisionNumber && $state eq "WV" ) { + my @divisionNumber = split /:/, $divisionNumber; + + #print $divisionNumber[3] . "\n"; + $csv .= "$divisionNumber[3],"; + } + else { + $csv .= ","; + } + } + + } + else { + $csv .= "$city $state $zip,"; + } + + if ($joinDate) { + $csv .= "$joinDate,"; + } + else { + $csv .= ","; + } + + print $csv . "\n"; + } + else { + if ( email($email) ) { + print "$firstName $lastName <$email>\n"; + } + } + } +} +close $fh; + +# skip over invalid and nonexistent email addresses +sub email { + + my @arg = @_; + + my $email = $arg[0]; + + if ( $email + && $email !~ +/^\w{1,2}\@aol.com|^\w{1,2}\@(em|m)ail.com|\@lwv.org|Email|(student|prando|ckarr)\@aol.com/ + ) + { + return 1; + } + else { + return 0; + } + +} + +sub help { + + my $help = "LLAW Google Civic Information API Query version 1 +Copyright (C) 2023 - by Jonathan Rosenbaum +This may be freely redistributed under the terms of the GNU General Public License + +Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate District for all LWVWV members) + $0 WV000 ./roster-file (show all information for members at large, but do not query Google) + $0 WV000 ./roster-file email (only show email addresses for members at large, but do not query Google) + +1st argument can be one of these types: + google (all members with senate/delegate district query) which prints out this csv data: + 'Name,Email,Phone,Address,Delegate District,Senate District,Join Date' + ALL (all members without senate/delegate district query) + League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson) + +2nd argument must be the location of the LWVWV roster file, and prints out this csv information: + Name,Email,Phone,Address,Join Date, + +3rd argument 'email' will only print out the email addresses, and only works with the ALL or League ID type argument + + +You will want to send results to a file. +Example: $0 google ./roster-file > 2023-districts +\n"; + + print $help; + +}