diff --git a/README.md b/README.md index b2ce228..3aa7821 100644 --- a/README.md +++ b/README.md @@ -153,4 +153,41 @@ This may be freely redistributed under the terms of the GNU General Public Licen 3rd argument 'email' will only print out the email addresses, and only works with the ALL or League ID type argument You will want to send results to a file. - Example: ./google-civic-api.pl google '*.csv' > 2023-districts \ No newline at end of file + Example: ./google-civic-api.pl google '*.csv' > 2023-districts + +## Automation + +### subscribe-members.sh + +Batch subscription example using [google-civic-api.pl](https://git.bikeshopi.dev/bike/lwvwv) to extract member data from LWVWV membership CSV files. This script processes membership exports and automatically subscribes members to mailing lists. + +**Prerequisites:** +- `google-civic-api.pl` must be in the same directory or PATH +- Membership CSV file from LWVWV portal + +**Usage:** +```bash +./subscribe-members.sh [list_id] +``` + +**Examples:** +```bash +# Using default list +./subscribe-members.sh league_membership.csv + +# Using specific list +./subscribe-members.sh league_membership.csv test.lists.example.org +``` + +**How it works:** +1. Runs `./google-civic-api.pl ALL email-csv` to extract emails +2. Converts output to CSV format: `email,first_name,last_name` +3. Sends batch subscribe request to connector + +**Configuration:** +Edit the script to set: +```bash +CONNECTOR_URL="https://mailman-connector.example.com" +CONNECTOR_PASSWORD="your_secret_password" +LIST_ID="test.lists.example.org" # Default list + diff --git a/download-roster.js b/download-roster.js index 9d08966..59d3d49 100755 --- a/download-roster.js +++ b/download-roster.js @@ -68,7 +68,7 @@ function sendErrorEmail(errorMessage) { const hostname = require('os').hostname(); const subject = `LWV Roster Download Failed on: ${hostname}`; - const body = `Error: ${errorMessage}\n\nTo fix this, please run: node save-session.js\n\nThen log in using the magic link method to refresh your session.`; + const body = `Error: ${errorMessage}\n\nTo fix this, please run: node save-session.js\n\nThen log in using the magic link method to refresh your session. If error persists, download-roster.js may need to be updated to relect changes on the portal page.`; // Build s-nail command - match user's working format exactly // Note: smtp value needs quotes because it contains ":" @@ -184,39 +184,106 @@ async function downloadRoster() { console.log('Looking for Export button...'); await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'membership-page.png') }); - // Try to find Export button/link - const exportBtn = await page.locator('button:has-text("Export")').first(); + // Try to find Export button (using ID for reliability) + console.log('Looking for Export button...'); + await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'membership-page.png') }); + const exportBtn = await page.locator('#league_membership_state_view-export-trigger, button:has-text("Export")').first(); + if (await exportBtn.count() > 0) { - console.log('Clicking Export...'); - await exportBtn.click(); - await page.waitForTimeout(3000); + // First, try to extract download URL directly from HTML (in case export was already generated) + console.log('Checking for existing download URL in page HTML...'); + const pageContent = await page.content(); + const downloadUrlMatch = pageContent.match(/href="(\/file\/download\/table_export[^"]+)"/); + let downloadUrl = downloadUrlMatch ? downloadUrlMatch[1] : null; + + if (downloadUrl) { + console.log('Found existing download URL in HTML:', downloadUrl); + } else { + // Two-step export process: + // Step 1: Click Export button to open modal + console.log('Clicking Export button to open modal...'); + await exportBtn.scrollIntoViewIfNeeded(); + await exportBtn.click({ force: true }); + await page.waitForTimeout(2000); + + // Step 2: Wait for and click "Start Export" button in modal + console.log('Looking for Start Export button in modal...'); + const startExportBtn = await page.locator('button[type="submit"]:has-text("Start Export"), button:has-text("Start Export")').first(); + + if (await startExportBtn.count() > 0) { + console.log('Clicking Start Export button...'); + await startExportBtn.click({ force: true }); + + // Step 3: Wait for export status section to appear + console.log('Waiting for export to complete...'); + try { + await page.waitForSelector('#league_membership_state_view-status-completed', { + state: 'visible', + timeout: 15000 + }); + console.log('Export status section is now visible'); + } catch (e) { + console.log('Timeout waiting for export status section...'); + } + + await page.waitForTimeout(2000); + } else { + console.log('Start Export button not found, trying alternative methods...'); + } + + // Try to extract URL from updated HTML + console.log('Checking for download URL after export...'); + const updatedContent = await page.content(); + const updatedMatch = updatedContent.match(/href="(\/file\/download\/table_export[^"]+)"/); + if (updatedMatch) { + downloadUrl = updatedMatch[1]; + console.log('Found download URL:', downloadUrl); + } + + // If still no URL, try to get it from the visible download link + if (!downloadUrl) { + const downloadLink = await page.locator('#league_membership_state_view-status-completed a[href*="/file/download"]').first(); + if (await downloadLink.count() > 0) { + downloadUrl = await downloadLink.getAttribute('href'); + console.log('Found download URL from link element:', downloadUrl); + } + } + } + await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'after-export-click.png') }); - // Look for download button - const downloadBtn = await page.locator('input[type="submit"][value="Download"]').first(); - if (await downloadBtn.count() > 0) { - console.log('Clicking Download...'); + if (downloadUrl) { + // Ensure URL is absolute + const fullUrl = downloadUrl.startsWith('http') ? downloadUrl : `https://portal.lwv.org${downloadUrl}`; + console.log('Download URL:', fullUrl); - const downloadPromise = page.waitForEvent('download', { timeout: 30000 }); - await downloadBtn.click(); + // Use Playwright's request API (automatically includes cookies) + console.log('Downloading via Playwright request API...'); + const response = await page.request.get(fullUrl); - const download = await downloadPromise; + if (!response.ok()) { + throw new Error(`Download failed with status: ${response.status()}`); + } + + // Save file const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); const filename = `league_membership_state_view_${timestamp}.csv`; const filepath = path.join(DOWNLOAD_DIR, filename); - await download.saveAs(filepath); + const buffer = await response.body(); + fs.writeFileSync(filepath, buffer); console.log(`Downloaded: ${filename}`); - + // Verify the file if (fs.existsSync(filepath)) { const stats = fs.statSync(filepath); console.log(`File size: ${stats.size} bytes`); } } else { - const error = 'Download button not found in export modal'; + const error = 'Download link not found after clicking Export'; console.log(error); + await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'download-not-found.png') }); sendErrorEmail(error); throw new Error(error); } diff --git a/google-civic-api.pl b/google-civic-api.pl index 32bfab0..615a595 100755 --- a/google-civic-api.pl +++ b/google-civic-api.pl @@ -72,8 +72,14 @@ else { } my $emailArg; +my $emailCsvArg; if ( $ARGV[2] ) { - $emailArg = $ARGV[2]; + if ( $ARGV[2] eq 'email-csv' ) { + $emailCsvArg = 1; + } + else { + $emailArg = $ARGV[2]; + } } # Open roster file .. hopefully columns remain the same, @@ -91,20 +97,20 @@ foreach my $file (@files) { # Read the first line my $localLeague; my $header = <$fh>; - print "Header: $header"; # Output the header for reference + print "Header: $header" unless $emailCsvArg; # Output the header for reference # Decide how to process if ( $header !~ /League ID/ ) { - print "Processing file: $file ... No League ID found, must be MAL\n"; + print "Processing file: $file ... No League ID found, must be MAL\n" unless $emailCsvArg; $localLeague = 0; # This is a Members At Large (MAL) file } else { - print "Processing file: $file ... League ID found\n"; + print "Processing file: $file ... League ID found\n" unless $emailCsvArg; $localLeague = 1; # This is local League file } my $csv = "Name,Email,Phone,Address,Delegate District,Senate District,League ID,Join Date\n"; - print $csv; + print $csv unless $emailCsvArg; $csv = ""; while ( my $line = <$fh> ) { @@ -120,32 +126,34 @@ foreach my $file (@files) { if ($localLeague) { # Local League file processing - # League ID is in column 11 - ( $leagueId = $fields[11] ) =~ s/"//g; # League ID - ( $phone = $fields[4] ) =~ s/"//g; # Phone - ( $email = $fields[3] ) =~ s/"//g; # Email - ( $joinDate = $fields[13] ) =~ s/"//g; # Join Date - ( $status = $fields[12] ) =~ s/"//g; # Status - field [12] for Local League + # Updated for new CSV header format (as of 2026-05-02) + # First Name[0],Preferred First Name[1],League Name[2],League ID[3],Status[4],Joined[5],Expiration[6],Unique Contact Id[7],Unique Account Id[8],Last Name[9],Email[10],Phone[11],Mailing Street[12],Mailing City[13],Mailing State[14],Mailing Postal Code[15],Mailing Country[16] + ( $leagueId = $fields[3] ) =~ s/"//g; # League ID [was 11] + ( $phone = $fields[11] ) =~ s/"//g; # Phone [was 4] + ( $email = $fields[10] ) =~ s/"//g; # Email [was 3] + ( $joinDate = $fields[5] ) =~ s/"//g; # Join Date [was 13] + ( $status = $fields[4] ) =~ s/"//g; # Status [was 12] } else { # Members At Large (MAL) file processing + # Updated for new CSV header format (as of 2026-05-02) # League ID is assumed to be WV000 for all MALs $leagueId = $stateLeagueID; # Default for MALs - ( $phone = $fields[3] ) =~ s/"//g; # Phone - ( $email = $fields[4] ) =~ s/"//g; # Email - ( $joinDate = $fields[10] ) =~ s/"//g; # Join Date - ( $status = $fields[13] ) =~ s/"//g; # Status - field [13] for MAL files + ( $phone = $fields[11] ) =~ s/"//g; # Phone [was 3] + ( $email = $fields[10] ) =~ s/"//g; # Email [was 4] + ( $joinDate = $fields[5] ) =~ s/"//g; # Join Date [was 10] + ( $status = $fields[4] ) =~ s/"//g; # Status [was 13] } - ( my $firstName = $fields[0] ) =~ s/"//g; # First Name - ( my $lastName = $fields[2] ) =~ s/"//g; # Last Name - ( my $street = $fields[5] ) =~ s/"//g; # Street - ( my $city = $fields[6] ) =~ s/"//g; # City - ( my $state = $fields[7] ) =~ s/"//g; # State - ( my $zip = $fields[8] ) =~ s/"//g; # Zip + ( my $firstName = $fields[0] ) =~ s/"//g; # First Name [unchanged] + ( my $lastName = $fields[9] ) =~ s/"//g; # Last Name [was 2] + ( my $street = $fields[12] ) =~ s/"//g; # Street [was 5] + ( my $city = $fields[13] ) =~ s/"//g; # City [was 6] + ( my $state = $fields[14] ) =~ s/"//g; # State [was 7] + ( my $zip = $fields[15] ) =~ s/"//g; # Zip [was 8] next if $street eq "Mailing Street"; # Filter for Primary status only (Primary, Primary - Life, etc.) - next unless $status =~ /^Primary.*/; + next unless $status =~ /^Primary.*|Lapsed/; if ( $leagueId !~ /^$STATE/ ) { next; @@ -162,7 +170,20 @@ foreach my $file (@files) { if ( $status ne "Inactive" ) { - if ( !$emailArg ) { + if ( $emailCsvArg ) { + # CSV format: email,first_name,last_name + if ( email($email) ) { + print "$email,$firstName,$lastName\n"; + } + } + elsif ( $emailArg ) { + # Original email format: First Last + if ( email($email) ) { + print "$firstName $lastName <$email>\n"; + } + } + else { + # Full CSV output with all fields # contact for people with email address if ( email($email) ) { @@ -277,11 +298,6 @@ foreach my $file (@files) { print $csv . "\n"; } - else { - if ( email($email) ) { - print "$firstName $lastName <$email>\n"; - } - } } } close $fh; @@ -327,6 +343,7 @@ generate an information file or email file. Usage: $0 google '*.csv' (queries Google Civic Api Delegate and Senate District for all LWVWV members) $0 ${STATE}000 '*.csv' (show all information for members at large, but do not query Google) $0 ${STATE}000 '*.csv' email (only show email addresses for members at large, but do not query Google) + $0 ${STATE}000 '*.csv' email-csv (only show email addresses in CSV format for members at large, but do not query Google) 1st argument can be one of these types: google (all members with senate/delegate district query) which prints out this csv data: @@ -338,6 +355,7 @@ Usage: $0 google '*.csv' (queries Google Civic Api Delegate and Senate Distric Name,Email,Phone,Address,Join Date, 3rd argument 'email' will only print out the email addresses, and only works with the ALL or League ID type argument + 3rd argument 'email-csv' will print out email addresses in CSV format (email,first_name,last_name), and only works with the ALL or League ID type argument You will want to send results to a file.