You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
261 lines
8.0 KiB
261 lines
8.0 KiB
#!/usr/bin/perl -w
|
|
use strict;
|
|
|
|
# Google Civic Information API
|
|
# Quota: Queries per minute
|
|
# Current limit: 151
|
|
#
|
|
# Use sleep to limit queries to 30 per minute
|
|
# Currently there are 187 members in the whole state
|
|
|
|
# commandline options: type roster-file email
|
|
# type:
|
|
# google (all members with senate/delegate district query)
|
|
# ALL (all members without senate/delegate district query)
|
|
# League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson)
|
|
# email (optional third arg):
|
|
# email (emails for specified type ALL (or google) or League ID)
|
|
|
|
# cpan App::cpanminus
|
|
# Now install any module you can find.
|
|
# cpanm Module::Name
|
|
use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required
|
|
use WWW::Curl::Simple;
|
|
use JSON;
|
|
use Data::Dumper;
|
|
use vars qw($key);
|
|
require "./env";
|
|
|
|
# Configuration - key for LWVWV
|
|
# put it in a file call env with one line $key = "thekeyfromgoogle"
|
|
my $googleApiKey = $key;
|
|
|
|
# Allow normal queries, too
|
|
my $searchType = $ARGV[0];
|
|
|
|
if ( !$ARGV[0] ) {
|
|
help();
|
|
exit;
|
|
}
|
|
else {
|
|
if ( $ARGV[0] !~ /google|ALL|WV000|WV102|WV103|WV107|WV112/ ) {
|
|
print "WRONG FIRST ARGUMENT\n";
|
|
help();
|
|
print "WRONG FIRST ARGUMENT\n";
|
|
exit;
|
|
}
|
|
}
|
|
|
|
my $emailArg;
|
|
if ( $ARGV[2] ) {
|
|
$emailArg = $ARGV[2];
|
|
}
|
|
|
|
# Open roster file .. hopefully columns remain the same
|
|
my $file;
|
|
$file = $ARGV[1];
|
|
my $fh;
|
|
$| = 1;
|
|
open $fh, "<", $file;
|
|
|
|
my $curl = WWW::Curl::Simple->new();
|
|
|
|
my $csv =
|
|
"Name,Email,Phone,Address,Join Date,Delegate District,Senate District,\n";
|
|
print $csv;
|
|
$csv = "";
|
|
while ( my $line = <$fh> ) {
|
|
|
|
# my @fields = split( /,/, $line );
|
|
# may be commas between those quotes
|
|
my @fields = $line =~ m/("[^"]+"|[^,]+)(?:,\s*)?/g;
|
|
( my $leagueId = $fields[0] ) =~ s/"//g; # League
|
|
( my $firstName = $fields[4] ) =~ s/"//g; # First Name
|
|
( my $lastName = $fields[5] ) =~ s/"//g; # Last Name
|
|
( my $status = $fields[6] ) =~ s/"//g; # Active
|
|
( my $joinDate = $fields[9] ) =~ s/"//g; # Join Date
|
|
( my $phone = $fields[10] ) =~ s/"//g; # Phone
|
|
( my $email = $fields[11] ) =~ s/"//g; # Email
|
|
( my $street = $fields[12] ) =~ s/"//g; # Street
|
|
( my $city = $fields[13] ) =~ s/"//g; # City
|
|
( my $state = $fields[14] ) =~ s/"//g; # State
|
|
( my $zip = $fields[16] ) =~ s/"//g; # Zip
|
|
next if $street eq "Mailing Street";
|
|
next if $leagueId !~ /^WV/;
|
|
|
|
if ( $searchType ne "google"
|
|
&& $searchType ne "ALL" )
|
|
{
|
|
next if $leagueId ne $searchType;
|
|
}
|
|
|
|
# no need for commas now
|
|
$street =~ s/,//g; # Street
|
|
|
|
if ( $status ne "Inactive" ) {
|
|
|
|
if ( !$emailArg ) {
|
|
|
|
# contact for people with email address
|
|
if ( email($email) ) {
|
|
$csv = "$firstName $lastName,";
|
|
$csv .= "$email,";
|
|
}
|
|
else {
|
|
$csv = "$firstName $lastName,";
|
|
$csv .= ",";
|
|
}
|
|
|
|
if ($phone) {
|
|
|
|
#print "PHONE: $fields[10]\n";
|
|
$csv .= "$phone,";
|
|
}
|
|
else {
|
|
#print "PHONE: none\n";
|
|
$csv .= ",";
|
|
}
|
|
|
|
#print "ADDRESS $fields[12] $fields[13] $fields[14] $fields[16]\n";
|
|
|
|
# Check for actual street
|
|
if ($street) {
|
|
|
|
# Correct known incorrect addresses
|
|
if ( $street eq "309 2nd St Altizer" ) {
|
|
$street =~ s/309 2nd St Altizer/309 2nd St/;
|
|
}
|
|
if ( $street eq "!87 Gallaher Street" ) {
|
|
$street =~ s/\!87 Gallaher Street/187 Gallaher Street/;
|
|
}
|
|
|
|
$csv .= "$street $city $state $zip,";
|
|
|
|
# url encoding
|
|
$street =~ s/#/%23/;
|
|
|
|
# Address query
|
|
my $address = "$street $city $state $zip\n";
|
|
|
|
if ( $searchType eq "google" ) {
|
|
|
|
# Delegate District Query
|
|
my $queryDelegateDistrict =
|
|
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorLowerBody&"
|
|
. "key=${googleApiKey}&"
|
|
. "address=${address}";
|
|
|
|
sleep 2;
|
|
my $res = $curl->get($queryDelegateDistrict);
|
|
my $content = decode_json( $res->content );
|
|
|
|
#print Dumper($content);
|
|
|
|
my $divisionNumber =
|
|
( ( keys %{ $content->{divisions} } )[0] );
|
|
if ( $divisionNumber && $state eq "WV" ) {
|
|
my @divisionNumber = split /:/, $divisionNumber;
|
|
|
|
#print $divisionNumber[3] . "\n";
|
|
$csv .= "$divisionNumber[3],";
|
|
}
|
|
else {
|
|
$csv .= ",";
|
|
}
|
|
|
|
# Senate District Query
|
|
my $querySenateDistrict =
|
|
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorUpperBody&"
|
|
. "key=${googleApiKey}&"
|
|
. "address=${address}";
|
|
|
|
sleep 2;
|
|
$res = $curl->get($querySenateDistrict);
|
|
$content = decode_json( $res->content );
|
|
|
|
# print Dumper($content);
|
|
$divisionNumber =
|
|
( ( keys %{ $content->{divisions} } )[0] );
|
|
if ( $divisionNumber && $state eq "WV" ) {
|
|
my @divisionNumber = split /:/, $divisionNumber;
|
|
|
|
#print $divisionNumber[3] . "\n";
|
|
$csv .= "$divisionNumber[3],";
|
|
}
|
|
else {
|
|
$csv .= ",";
|
|
}
|
|
}
|
|
|
|
}
|
|
else {
|
|
$csv .= "$city $state $zip,";
|
|
}
|
|
|
|
if ($joinDate) {
|
|
$csv .= "$joinDate,";
|
|
}
|
|
else {
|
|
$csv .= ",";
|
|
}
|
|
|
|
print $csv . "\n";
|
|
}
|
|
else {
|
|
if ( email($email) ) {
|
|
print "$firstName $lastName <$email>\n";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
close $fh;
|
|
|
|
# skip over invalid and nonexistent email addresses
|
|
sub email {
|
|
|
|
my @arg = @_;
|
|
|
|
my $email = $arg[0];
|
|
|
|
if ( $email
|
|
&& $email !~
|
|
/^\w{1,2}\@aol.com|^\w{1,2}\@(em|m)ail.com|\@lwv.org|Email|(student|prando|ckarr)\@aol.com/
|
|
)
|
|
{
|
|
return 1;
|
|
}
|
|
else {
|
|
return 0;
|
|
}
|
|
|
|
}
|
|
|
|
sub help {
|
|
|
|
my $help = "LLAW Google Civic Information API Query version 1
|
|
Copyright (C) 2023 - by Jonathan Rosenbaum <freesource\@freesoftwarepc.com>
|
|
This may be freely redistributed under the terms of the GNU General Public License
|
|
|
|
Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate District for all LWVWV members)
|
|
$0 WV000 ./roster-file email (show all information for LWVWV members, but do not query Google)
|
|
$0 WV000 ./roster-file email (only show email addresses for members at large, but do not query Google)
|
|
|
|
1st argument can be one of these types:
|
|
google (all members with senate/delegate district query) which prints out this csv data:
|
|
'Name,Email,Phone,Address,Delegate District,Senate District,Join Date'
|
|
ALL (all members without senate/delegate district query)
|
|
League ID: WV000 (members at large) WV102 (Huntington) WV103 (Morgantown-Monogalia) WV107 (Wood County) WV112 (Jefferson)
|
|
|
|
2nd argument must be the location of the LWVWV roster file, and prints out this csv information:
|
|
Name,Email,Phone,Address,Join Date,
|
|
|
|
3rd argument 'email' will only print out the email addresses, and only works with the ALL or League ID type argument
|
|
|
|
|
|
You will want to send results to a file.
|
|
Example: $0 google ./roster-file > 2023-districts
|
|
\n";
|
|
|
|
print $help;
|
|
|
|
}
|
|
|