#!/usr/bin/perl -w use strict; # Google Civic Information API # Quota: Queries per minute # Current limit: 151 # # Use sleep to limit queries to 30 per minute # Currently there are 187 members in the whole state # commandline options: type roster-file email # type: # google (all members with senate/delegate district query) # ALL (all members without senate/delegate district query) # League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson) # email (optional third arg): # email (emails for specified type ALL (or google) or League ID) # cpan App::cpanminus # Now install any module you can find. # cpanm Module::Name use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required use WWW::Curl::Simple; use JSON; use Data::Dumper; use vars qw($key); require "./env"; # Configuration - key for LWVWV # put it in a file call env with one line $key = "thekeyfromgoogle" my $googleApiKey = $key; # Allow normal queries, too my $searchType = $ARGV[0]; if ( !$ARGV[0] ) { help(); exit; } else { if ( $ARGV[0] !~ /google|ALL|WV000|WV102|WV103|WV107|WV112/ ) { print "WRONG FIRST ARGUMENT\n"; help(); print "WRONG FIRST ARGUMENT\n"; exit; } } my $emailArg; if ( $ARGV[2] ) { $emailArg = $ARGV[2]; } # Open roster file .. hopefully columns remain the same my $file; $file = $ARGV[1]; my $fh; $| = 1; open $fh, "<", $file; my $curl = WWW::Curl::Simple->new(); my $csv = "Name,Email,Phone,Address,Join Date,Delegate District,Senate District,\n"; print $csv; $csv = ""; while ( my $line = <$fh> ) { # my @fields = split( /,/, $line ); # may be commas between those quotes my @fields = $line =~ m/("[^"]+"|[^,]+)(?:,\s*)?/g; ( my $leagueId = $fields[0] ) =~ s/"//g; # League ( my $firstName = $fields[4] ) =~ s/"//g; # First Name ( my $lastName = $fields[5] ) =~ s/"//g; # Last Name ( my $status = $fields[6] ) =~ s/"//g; # Active ( my $joinDate = $fields[9] ) =~ s/"//g; # Join Date ( my $phone = $fields[10] ) =~ s/"//g; # Phone ( my $email = $fields[11] ) =~ s/"//g; # Email ( my $street = $fields[12] ) =~ s/"//g; # Street ( my $city = $fields[13] ) =~ s/"//g; # City ( my $state = $fields[14] ) =~ s/"//g; # State ( my $zip = $fields[16] ) =~ s/"//g; # Zip next if $street eq "Mailing Street"; next if $leagueId !~ /^WV/; if ( $searchType ne "google" && $searchType ne "ALL" ) { next if $leagueId ne $searchType; } # no need for commas now $street =~ s/,//g; # Street if ( $status ne "Inactive" ) { if ( !$emailArg ) { # contact for people with email address if ( email($email) ) { $csv = "$firstName $lastName,"; $csv .= "$email,"; } else { $csv = "$firstName $lastName,"; $csv .= ","; } if ($phone) { #print "PHONE: $fields[10]\n"; $csv .= "$phone,"; } else { #print "PHONE: none\n"; $csv .= ","; } #print "ADDRESS $fields[12] $fields[13] $fields[14] $fields[16]\n"; # Check for actual street if ($street) { # Correct known incorrect addresses if ( $street eq "309 2nd St Altizer" ) { $street =~ s/309 2nd St Altizer/309 2nd St/; } if ( $street eq "!87 Gallaher Street" ) { $street =~ s/\!87 Gallaher Street/187 Gallaher Street/; } $csv .= "$street $city $state $zip,"; # url encoding $street =~ s/#/%23/; # Address query my $address = "$street $city $state $zip\n"; if ( $searchType eq "google" ) { # Delegate District Query my $queryDelegateDistrict = "https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorLowerBody&" . "key=${googleApiKey}&" . "address=${address}"; sleep 2; my $res = $curl->get($queryDelegateDistrict); my $content = decode_json( $res->content ); #print Dumper($content); my $divisionNumber = ( ( keys %{ $content->{divisions} } )[0] ); if ( $divisionNumber && $state eq "WV" ) { my @divisionNumber = split /:/, $divisionNumber; #print $divisionNumber[3] . "\n"; $csv .= "$divisionNumber[3],"; } else { $csv .= ","; } # Senate District Query my $querySenateDistrict = "https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorUpperBody&" . "key=${googleApiKey}&" . "address=${address}"; sleep 2; $res = $curl->get($querySenateDistrict); $content = decode_json( $res->content ); # print Dumper($content); $divisionNumber = ( ( keys %{ $content->{divisions} } )[0] ); if ( $divisionNumber && $state eq "WV" ) { my @divisionNumber = split /:/, $divisionNumber; #print $divisionNumber[3] . "\n"; $csv .= "$divisionNumber[3],"; } else { $csv .= ","; } } } else { $csv .= "$city $state $zip,"; } if ($joinDate) { $csv .= "$joinDate,"; } else { $csv .= ","; } print $csv . "\n"; } else { if ( email($email) ) { print "$firstName $lastName <$email>\n"; } } } } close $fh; # skip over invalid and nonexistent email addresses sub email { my @arg = @_; my $email = $arg[0]; if ( $email && $email !~ /^\w{1,2}\@aol.com|^\w{1,2}\@(em|m)ail.com|\@lwv.org|Email|(student|prando|ckarr)\@aol.com/ ) { return 1; } else { return 0; } } sub help { my $help = "LLAW Google Civic Information API Query version 1 Copyright (C) 2023 - by Jonathan Rosenbaum This may be freely redistributed under the terms of the GNU General Public License Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate District for all LWVWV members) $0 WV000 ./roster-file email (show all information for LWVWV members, but do not query Google) $0 WV000 ./roster-file email (only show email addresses for members at large, but do not query Google) 1st argument can be one of these types: google (all members with senate/delegate district query) which prints out this csv data: 'Name,Email,Phone,Address,Delegate District,Senate District,Join Date' ALL (all members without senate/delegate district query) League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson) 2nd argument must be the location of the LWVWV roster file, and prints out this csv information: Name,Email,Phone,Address,Join Date, 3rd argument 'email' will only print out the email addresses, and only works with the ALL or League ID type argument You will want to send results to a file. Example: $0 google ./roster-file > 2023-districts \n"; print $help; }