Updates to new format for https://portal.lwv.org. Now requires two different csv files.
This commit is contained in:
parent
40fe62fbdd
commit
778d10b7bf
@ -1,6 +1,26 @@
|
||||
#!/usr/bin/perl -w
|
||||
use strict;
|
||||
|
||||
# PORTAL CHANGES - assumes you have roster permission for the State League group
|
||||
# Since the portal came online, you now need to process two different types of cvs files: One for all local Leagues (Local Leagues) & one for the MALs (Roster)
|
||||
# * * * * * * * * * *
|
||||
# LEAGUE - First Name Preferred First Name Last Name Email Phone Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country League Name League ID Status Original Join Date Expiration Date Unique Contact Id Unique Account Id
|
||||
# (diff) (diff)
|
||||
# MAL - First Name Preferred First Name Last Name Phone Email Mailing Street Mailing City Mailing State Mailing Postal Code Mailing Country Original Join Date Expiration Date Last Login Date Unique Contact Id Unique Account Id
|
||||
|
||||
# Required local Leagues MAL
|
||||
# Preferred First Name [1] [1]
|
||||
# Last Name [2] [2]
|
||||
# Email [3] [4]
|
||||
# Phone [4] [3]
|
||||
# Mailing Street [5] [5]
|
||||
# Mailing City [6] [6]
|
||||
# Mailing State [7] [7]
|
||||
# Mailing Postal Code [8] [8]
|
||||
# League ID [11] Assumed
|
||||
# Status [12] not used not used
|
||||
# Original Join Date [13] [10]
|
||||
|
||||
# Google Civic Information API
|
||||
# Quota: Queries per minute
|
||||
# Current limit: 151
|
||||
@ -19,8 +39,9 @@ use strict;
|
||||
# cpan App::cpanminus
|
||||
# Now install any module you can find.
|
||||
# cpanm Module::Name
|
||||
use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required
|
||||
use lib qw(/home/jr/perl5/lib/perl5); # adjust perl lib as required
|
||||
use WWW::Curl::Simple;
|
||||
use File::Glob ':glob';
|
||||
use JSON;
|
||||
use Data::Dumper;
|
||||
use vars qw($key);
|
||||
@ -33,6 +54,7 @@ my $googleApiKey = $key;
|
||||
# Allow normal queries, too
|
||||
my $searchType = $ARGV[0];
|
||||
|
||||
my @files;
|
||||
if ( !$ARGV[0] ) {
|
||||
help();
|
||||
exit;
|
||||
@ -44,6 +66,8 @@ else {
|
||||
print "WRONG FIRST ARGUMENT\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
@files = bsd_glob( $ARGV[1] ); # Expand wildcard patterns (e.g., *.csv)
|
||||
}
|
||||
|
||||
my $emailArg;
|
||||
@ -53,168 +77,204 @@ if ( $ARGV[2] ) {
|
||||
|
||||
# Open roster file .. hopefully columns remain the same,
|
||||
# but @fields can be adjusted as required.
|
||||
my $file;
|
||||
$file = $ARGV[1];
|
||||
my $fh;
|
||||
$| = 1;
|
||||
open $fh, "<", $file;
|
||||
foreach my $file (@files) {
|
||||
|
||||
my $curl = WWW::Curl::Simple->new();
|
||||
#
|
||||
# $file = $ARGV[1];
|
||||
my $fh;
|
||||
$| = 1;
|
||||
open $fh, "<", $file;
|
||||
|
||||
my $csv =
|
||||
"Name,Email,Phone,Address,Delegate District,Senate District,League ID,Join Date\n";
|
||||
print $csv;
|
||||
$csv = "";
|
||||
while ( my $line = <$fh> ) {
|
||||
my $curl = WWW::Curl::Simple->new();
|
||||
|
||||
# my @fields = split( /,/, $line );
|
||||
# may be commas between those quotes
|
||||
# pulled this regexp from https://stackoverflow.com/questions/18144431/regex-to-split-a-csv (solution 45)
|
||||
my @fields = $line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g; #m/("[^"]+"|[^,]+)(?:,\s*)?/g;
|
||||
# print $#fields . "\n";
|
||||
( my $leagueId = $fields[1] ) =~ s/"//g; # League
|
||||
( my $firstName = $fields[4] ) =~ s/"//g; # First Name
|
||||
( my $lastName = $fields[5] ) =~ s/"//g; # Last Name
|
||||
( my $status = $fields[7] ) =~ s/"//g; # Active
|
||||
( my $joinDate = $fields[10] ) =~ s/"//g; # Join Date
|
||||
( my $phone = $fields[11] ) =~ s/"//g; # Phone
|
||||
( my $email = $fields[12] ) =~ s/"//g; # Email
|
||||
( my $street = $fields[13] ) =~ s/"//g; # Street
|
||||
( my $city = $fields[14] ) =~ s/"//g; # City
|
||||
( my $state = $fields[15] ) =~ s/"//g; # State
|
||||
( my $zip = $fields[17] ) =~ s/"//g; # Zip
|
||||
next if $street eq "Mailing Street";
|
||||
if ( $leagueId !~ /^WV/ && $leagueId ne "LWV Of West Virginia") {
|
||||
next;
|
||||
# Read the first line
|
||||
my $localLeague;
|
||||
my $header = <$fh>;
|
||||
print "Header: $header"; # Output the header for reference
|
||||
# Decide how to process
|
||||
if ( $header !~ /League ID/ ) {
|
||||
print "Processing file: $file ... No League ID found, must be MAL\n";
|
||||
$localLeague = 0; # This is a Members At Large (MAL) file
|
||||
}
|
||||
else {
|
||||
print "Processing file: $file ... League ID found\n";
|
||||
$localLeague = 1; # This is local League file
|
||||
}
|
||||
|
||||
if ( $searchType ne "google"
|
||||
&& $searchType ne "ALL" )
|
||||
{
|
||||
next if $leagueId ne $searchType;
|
||||
}
|
||||
my $csv =
|
||||
"Name,Email,Phone,Address,Delegate District,Senate District,League ID,Join Date\n";
|
||||
print $csv;
|
||||
$csv = "";
|
||||
|
||||
# no need for commas now
|
||||
$street =~ s/,//g; # Street
|
||||
while ( my $line = <$fh> ) {
|
||||
|
||||
if ( $status ne "Inactive" ) {
|
||||
# my @fields = split( /,/, $line );
|
||||
# may be commas between those quotes
|
||||
# pulled this regexp from https://stackoverflow.com/questions/18144431/regex-to-split-a-csv (solution 45)
|
||||
my @fields =
|
||||
$line =~ m/(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))/g
|
||||
; #m/("[^"]+"|[^,]+)(?:,\s*)?/g;
|
||||
# print $#fields . "\n";
|
||||
my $leagueId, my $phone, my $email, my $joinDate;
|
||||
if ($localLeague) {
|
||||
|
||||
if ( !$emailArg ) {
|
||||
|
||||
# contact for people with email address
|
||||
if ( email($email) ) {
|
||||
$csv = "$firstName $lastName,";
|
||||
$csv .= "$email,";
|
||||
}
|
||||
else {
|
||||
$csv = "$firstName $lastName,";
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
if ($phone) {
|
||||
|
||||
#print "PHONE: $fields[10]\n";
|
||||
$csv .= "$phone,";
|
||||
}
|
||||
else {
|
||||
#print "PHONE: none\n";
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
#print "ADDRESS $fields[12] $fields[13] $fields[14] $fields[16]\n";
|
||||
|
||||
# Check for actual street
|
||||
if ($street) {
|
||||
|
||||
# Correct known incorrect addresses
|
||||
if ( $street eq "309 2nd St Altizer" ) {
|
||||
$street =~ s/309 2nd St Altizer/309 2nd St/;
|
||||
}
|
||||
if ( $street eq "!87 Gallaher Street" ) {
|
||||
$street =~ s/\!87 Gallaher Street/187 Gallaher Street/;
|
||||
}
|
||||
|
||||
$csv .= "$street $city $state $zip,";
|
||||
|
||||
# url encoding
|
||||
$street =~ s/#/%23/;
|
||||
|
||||
# Address query
|
||||
my $address = "$street $city $state $zip\n";
|
||||
|
||||
if ( $searchType eq "google" ) {
|
||||
|
||||
# Delegate District Query
|
||||
my $queryDelegateDistrict =
|
||||
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorLowerBody&"
|
||||
. "key=${googleApiKey}&"
|
||||
. "address=${address}";
|
||||
|
||||
sleep 2;
|
||||
my $res = $curl->get($queryDelegateDistrict);
|
||||
my $content = decode_json( $res->content );
|
||||
|
||||
#print Dumper($content);
|
||||
|
||||
my $divisionNumber =
|
||||
( ( keys %{ $content->{divisions} } )[0] );
|
||||
if ( $divisionNumber && $state eq "WV" ) {
|
||||
my @divisionNumber = split /:/, $divisionNumber;
|
||||
|
||||
#print $divisionNumber[3] . "\n";
|
||||
$csv .= "$divisionNumber[3],";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
# Senate District Query
|
||||
my $querySenateDistrict =
|
||||
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorUpperBody&"
|
||||
. "key=${googleApiKey}&"
|
||||
. "address=${address}";
|
||||
|
||||
sleep 2;
|
||||
$res = $curl->get($querySenateDistrict);
|
||||
$content = decode_json( $res->content );
|
||||
|
||||
# print Dumper($content);
|
||||
$divisionNumber =
|
||||
( ( keys %{ $content->{divisions} } )[0] );
|
||||
if ( $divisionNumber && $state eq "WV" ) {
|
||||
my @divisionNumber = split /:/, $divisionNumber;
|
||||
|
||||
#print $divisionNumber[3] . "\n";
|
||||
$csv .= "$divisionNumber[3],";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
$csv .= "$leagueId,";
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
$csv .= "$city $state $zip,";
|
||||
}
|
||||
|
||||
if ($joinDate) {
|
||||
$csv .= "$joinDate,";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
print $csv . "\n";
|
||||
# Local League file processing
|
||||
# League ID is in column 11
|
||||
( $leagueId = $fields[11] ) =~ s/"//g; # League ID
|
||||
( $phone = $fields[4] ) =~ s/"//g; # Phone
|
||||
( $email = $fields[3] ) =~ s/"//g; # Email
|
||||
( $joinDate = $fields[13] ) =~ s/"//g; # Join Date
|
||||
}
|
||||
else {
|
||||
if ( email($email) ) {
|
||||
print "$firstName $lastName <$email>\n";
|
||||
# Members At Large (MAL) file processing
|
||||
# League ID is assumed to be WV000 for all MALs
|
||||
$leagueId = "WV000"; # Default for MALs
|
||||
( $phone = $fields[3] ) =~ s/"//g; # Phone
|
||||
( $email = $fields[4] ) =~ s/"//g; # Email
|
||||
( $joinDate = $fields[10] ) =~ s/"//g; # Join Date
|
||||
}
|
||||
( my $firstName = $fields[0] ) =~ s/"//g; # First Name
|
||||
( my $lastName = $fields[2] ) =~ s/"//g; # Last Name
|
||||
( my $street = $fields[5] ) =~ s/"//g; # Street
|
||||
( my $city = $fields[6] ) =~ s/"//g; # City
|
||||
( my $state = $fields[7] ) =~ s/"//g; # State
|
||||
( my $zip = $fields[8] ) =~ s/"//g; # Zip
|
||||
my $status = "true";
|
||||
next if $street eq "Mailing Street";
|
||||
|
||||
if ( $leagueId !~ /^WV/ && $leagueId ne "LWV Of West Virginia" ) {
|
||||
next;
|
||||
}
|
||||
|
||||
if ( $searchType ne "google"
|
||||
&& $searchType ne "ALL" )
|
||||
{
|
||||
next if $leagueId ne $searchType;
|
||||
}
|
||||
|
||||
# no need for commas now
|
||||
$street =~ s/,//g; # Street
|
||||
|
||||
if ( $status ne "Inactive" ) {
|
||||
|
||||
if ( !$emailArg ) {
|
||||
|
||||
# contact for people with email address
|
||||
if ( email($email) ) {
|
||||
$csv = "$firstName $lastName,";
|
||||
$csv .= "$email,";
|
||||
}
|
||||
else {
|
||||
$csv = "$firstName $lastName,";
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
if ($phone) {
|
||||
|
||||
#print "PHONE: $fields[10]\n";
|
||||
$csv .= "$phone,";
|
||||
}
|
||||
else {
|
||||
#print "PHONE: none\n";
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
#print "ADDRESS $fields[12] $fields[13] $fields[14] $fields[16]\n";
|
||||
|
||||
# Check for actual street
|
||||
if ($street) {
|
||||
|
||||
# Correct known incorrect addresses
|
||||
if ( $street eq "309 2nd St Altizer" ) {
|
||||
$street =~ s/309 2nd St Altizer/309 2nd St/;
|
||||
}
|
||||
if ( $street eq "!87 Gallaher Street" ) {
|
||||
$street =~ s/\!87 Gallaher Street/187 Gallaher Street/;
|
||||
}
|
||||
|
||||
$csv .= "$street $city $state $zip,";
|
||||
|
||||
# url encoding
|
||||
$street =~ s/#/%23/;
|
||||
|
||||
# Address query
|
||||
my $address = "$street $city $state $zip\n";
|
||||
|
||||
if ( $searchType eq "google" ) {
|
||||
|
||||
# Delegate District Query
|
||||
my $queryDelegateDistrict =
|
||||
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorLowerBody&"
|
||||
. "key=${googleApiKey}&"
|
||||
. "address=${address}";
|
||||
|
||||
sleep 2;
|
||||
my $res = $curl->get($queryDelegateDistrict);
|
||||
my $content = decode_json( $res->content );
|
||||
|
||||
#print Dumper($content);
|
||||
|
||||
my $divisionNumber =
|
||||
( ( keys %{ $content->{divisions} } )[0] );
|
||||
if ( $divisionNumber && $state eq "WV" ) {
|
||||
my @divisionNumber = split /:/, $divisionNumber;
|
||||
|
||||
#print $divisionNumber[3] . "\n";
|
||||
$csv .= "$divisionNumber[3],";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
# Senate District Query
|
||||
my $querySenateDistrict =
|
||||
"https://www.googleapis.com/civicinfo/v2/representatives?includeOffices=true&levels=administrativeArea1&roles=legislatorUpperBody&"
|
||||
. "key=${googleApiKey}&"
|
||||
. "address=${address}";
|
||||
|
||||
sleep 2;
|
||||
$res = $curl->get($querySenateDistrict);
|
||||
$content = decode_json( $res->content );
|
||||
|
||||
# print Dumper($content);
|
||||
$divisionNumber =
|
||||
( ( keys %{ $content->{divisions} } )[0] );
|
||||
if ( $divisionNumber && $state eq "WV" ) {
|
||||
my @divisionNumber = split /:/, $divisionNumber;
|
||||
|
||||
#print $divisionNumber[3] . "\n";
|
||||
$csv .= "$divisionNumber[3],";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
$csv .= "$leagueId,";
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
$csv .= "$city $state $zip,";
|
||||
}
|
||||
|
||||
if ($joinDate) {
|
||||
$csv .= "$joinDate,";
|
||||
}
|
||||
else {
|
||||
$csv .= ",";
|
||||
}
|
||||
|
||||
print $csv . "\n";
|
||||
}
|
||||
else {
|
||||
if ( email($email) ) {
|
||||
print "$firstName $lastName <$email>\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
close $fh;
|
||||
|
||||
}
|
||||
close $fh;
|
||||
|
||||
# skip over invalid and nonexistent email addresses
|
||||
sub email {
|
||||
@ -242,9 +302,15 @@ sub help {
|
||||
Copyright (C) 2023 - by Jonathan Rosenbaum <freesource\@freesoftwarepc.com>
|
||||
This may be freely redistributed under the terms of the GNU General Public License
|
||||
|
||||
Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate District for all LWVWV members)
|
||||
$0 WV000 ./roster-file (show all information for members at large, but do not query Google)
|
||||
$0 WV000 ./roster-file email (only show email addresses for members at large, but do not query Google)
|
||||
Note: Since https://portal.lwvwv.org came online, two csv files are now available, which should both be processed:
|
||||
1. Local Leagues (Local Leagues) - which has all the local League members
|
||||
2. MAL (Roster) - which has all the members at large
|
||||
Depending on your request, this script will process either of those files, and query the Google Civic Information API for the Delegate and Senate Districts,
|
||||
generate an information file or email file.
|
||||
|
||||
Usage: $0 google '*.csv' (queries Google Civic Api Delegate and Senate District for all LWVWV members)
|
||||
$0 WV000 '*.csv' (show all information for members at large, but do not query Google)
|
||||
$0 WV000 '*.csv' email (only show email addresses for members at large, but do not query Google)
|
||||
|
||||
1st argument can be one of these types:
|
||||
google (all members with senate/delegate district query) which prints out this csv data:
|
||||
@ -259,7 +325,7 @@ Usage: $0 google ./roster-file (queries Google Civic Api Delegate and Senate D
|
||||
|
||||
|
||||
You will want to send results to a file.
|
||||
Example: $0 google ./roster-file > 2023-districts
|
||||
Example: $0 google '*.csv' > 2023-districts
|
||||
\n";
|
||||
|
||||
print $help;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user