#!/usr/bin/env node const { chromium } = require('playwright'); const fs = require('fs'); const path = require('path'); const { execSync } = require('child_process'); const DOWNLOAD_DIR = path.dirname(__filename); const SESSION_FILE = path.join(DOWNLOAD_DIR, '.session.json'); const ENV_FILE = path.join(DOWNLOAD_DIR, 'env'); // Read configuration from env file function loadConfig() { const content = fs.readFileSync(ENV_FILE, 'utf8'); const config = {}; // Match $VAR = "value" patterns const matches = content.matchAll(/\$(\w+)\s*=\s*"([^"]*)"/g); for (const match of matches) { config[match[1]] = match[2]; } return config; } const config = loadConfig(); const PORTAL_URL = config.PORTAL_URL || 'https://portal.lwv.org'; const MEMBERSHIP_URL = config.MEMBERSHIP_URL || 'https://portal.lwv.org/groups/53a93df1-9d4884/league_membership_state_view'; // Email config const SMTP_HOST = config.SMTP_HOST || ''; const SMTP_PORT = config.SMTP_PORT || '587'; const SMTP_USE_STARTTLS = config.SMTP_USE_STARTTLS || 'true'; const SMTP_AUTH = config.SMTP_AUTH || 'login'; const SMTP_USER = config.SMTP_USER || ''; const SMTP_PASSWORD = config.SMTP_PASSWORD || ''; const SSL_VERIFY_IGNORE = config.SSL_VERIFY_IGNORE || 'ignore'; const EMAIL_FROM = config.EMAIL_FROM || ''; const EMAIL_TO = config.EMAIL_TO || ''; function loadSession() { if (!fs.existsSync(SESSION_FILE)) { return null; } try { const sessionData = JSON.parse(fs.readFileSync(SESSION_FILE, 'utf8')); // Check if expired if (new Date(sessionData.expiresAt) < new Date()) { console.log('Session expired'); return null; } return sessionData; } catch (e) { console.error('Error loading session:', e.message); return null; } } function sendErrorEmail(errorMessage) { if (!SMTP_HOST || !EMAIL_TO) { console.log('Email not configured, skipping notification'); return; } const hostname = require('os').hostname(); const subject = `LWV Roster Download Failed on: ${hostname}`; const body = `Error: ${errorMessage}\n\nTo fix this, please run: node save-session.js\n\nThen log in using the magic link method to refresh your session. If error persists, download-roster.js may need to be updated to relect changes on the portal page.`; // Build s-nail command - match user's working format exactly // Note: smtp value needs quotes because it contains ":" const snaicmd = `s-nail -r "${EMAIL_FROM}" -s "${subject}" -S smtp="${SMTP_HOST}:${SMTP_PORT}" -S smtp-use-starttls -S smtp-auth=${SMTP_AUTH} -S smtp-auth-user="${SMTP_USER}" -S smtp-auth-password="${SMTP_PASSWORD}" -S ssl-verify=${SSL_VERIFY_IGNORE} ${EMAIL_TO}`; // Write body to temp file to avoid shell escaping issues const tempFile = '/tmp/lwv-email-body.txt'; fs.writeFileSync(tempFile, body); try { execSync(`cat ${tempFile} | ${snaicmd}`, { stdio: 'pipe' }); console.log('Error notification email sent'); fs.unlinkSync(tempFile); } catch (e) { console.log('Failed to send error email:', e.message); try { fs.unlinkSync(tempFile); } catch {} } } async function downloadRoster() { // Try to load saved session const sessionData = loadSession(); console.log('Starting browser...'); const browser = await chromium.launch({ headless: true, downloadsDir: DOWNLOAD_DIR }); let context; let page; // If we have a valid session, use it if (sessionData) { console.log('Using saved session from', sessionData.savedAt); // Use storageState if available (includes cookies and localStorage) if (sessionData.storageState) { console.log('Using storageState to restore session...'); context = await browser.newContext({ storageState: sessionData.storageState }); } else { context = await browser.newContext(); await context.addCookies(sessionData.cookies); } } else { const error = 'No valid session found. Run save-session.js first to create one.'; console.log(error); sendErrorEmail(error); await browser.close(); process.exit(1); } // Debug: check what cookies were added const addedCookies = await context.cookies(); console.log('Browser has', addedCookies.length, 'cookies'); page = await context.newPage(); try { // Navigate to portal home first to establish session console.log('Navigating to portal home...'); await page.goto(PORTAL_URL, { waitUntil: 'networkidle', timeout: 60000 }); await page.waitForTimeout(2000); console.log('Home URL:', page.url()); // Check if still on login page const homePageText = await page.textContent('body').catch(() => ''); console.log('Home page contains login:', homePageText.includes('Continue with email')); // If still on login page, session is definitely invalid if (homePageText.includes('Continue with email')) { const error = 'Session has expired - redirected to login page. Run save-session.js to refresh.'; console.log(error); sendErrorEmail(error); await browser.close(); process.exit(1); } // Now navigate to membership page console.log('Navigating to membership page...'); await page.goto(MEMBERSHIP_URL, { waitUntil: 'networkidle', timeout: 60000 }); await page.waitForTimeout(3000); console.log('URL:', page.url()); // Debug: print page title console.log('Page title:', await page.title()); // Debug: check if we're on login page const pageContent = await page.content(); const pageText = await page.textContent('body').catch(() => ''); console.log('Page text length:', pageText.length); console.log('Contains Continue with email:', pageText.includes('Continue with email')); // Verify we're on the right page - check for login page elements const isLoginPage = pageText.includes('Continue with email') || pageText.includes('Continue with password') || pageText.includes('Log in'); if (isLoginPage) { const error = 'Session has expired - redirected to login page. Run save-session.js to refresh.'; console.log(error); sendErrorEmail(error); await browser.close(); process.exit(1); } // Look for Export button console.log('Looking for Export button...'); await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'membership-page.png') }); // Try to find Export button (using ID for reliability) console.log('Looking for Export button...'); await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'membership-page.png') }); const exportBtn = await page.locator('#league_membership_state_view-export-trigger, button:has-text("Export")').first(); if (await exportBtn.count() > 0) { // First, try to extract download URL directly from HTML (in case export was already generated) console.log('Checking for existing download URL in page HTML...'); const pageContent = await page.content(); const downloadUrlMatch = pageContent.match(/href="(\/file\/download\/table_export[^"]+)"/); let downloadUrl = downloadUrlMatch ? downloadUrlMatch[1] : null; if (downloadUrl) { console.log('Found existing download URL in HTML:', downloadUrl); } else { // Two-step export process: // Step 1: Click Export button to open modal console.log('Clicking Export button to open modal...'); await exportBtn.scrollIntoViewIfNeeded(); await exportBtn.click({ force: true }); await page.waitForTimeout(2000); // Step 2: Wait for and click "Start Export" button in modal console.log('Looking for Start Export button in modal...'); const startExportBtn = await page.locator('button[type="submit"]:has-text("Start Export"), button:has-text("Start Export")').first(); if (await startExportBtn.count() > 0) { console.log('Clicking Start Export button...'); await startExportBtn.click({ force: true }); // Step 3: Wait for export status section to appear console.log('Waiting for export to complete...'); try { await page.waitForSelector('#league_membership_state_view-status-completed', { state: 'visible', timeout: 15000 }); console.log('Export status section is now visible'); } catch (e) { console.log('Timeout waiting for export status section...'); } await page.waitForTimeout(2000); } else { console.log('Start Export button not found, trying alternative methods...'); } // Try to extract URL from updated HTML console.log('Checking for download URL after export...'); const updatedContent = await page.content(); const updatedMatch = updatedContent.match(/href="(\/file\/download\/table_export[^"]+)"/); if (updatedMatch) { downloadUrl = updatedMatch[1]; console.log('Found download URL:', downloadUrl); } // If still no URL, try to get it from the visible download link if (!downloadUrl) { const downloadLink = await page.locator('#league_membership_state_view-status-completed a[href*="/file/download"]').first(); if (await downloadLink.count() > 0) { downloadUrl = await downloadLink.getAttribute('href'); console.log('Found download URL from link element:', downloadUrl); } } } await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'after-export-click.png') }); if (downloadUrl) { // Ensure URL is absolute const fullUrl = downloadUrl.startsWith('http') ? downloadUrl : `https://portal.lwv.org${downloadUrl}`; console.log('Download URL:', fullUrl); // Use Playwright's request API (automatically includes cookies) console.log('Downloading via Playwright request API...'); const response = await page.request.get(fullUrl); if (!response.ok()) { throw new Error(`Download failed with status: ${response.status()}`); } // Save file const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); const filename = `league_membership_state_view_${timestamp}.csv`; const filepath = path.join(DOWNLOAD_DIR, filename); const buffer = await response.body(); fs.writeFileSync(filepath, buffer); console.log(`Downloaded: ${filename}`); // Verify the file if (fs.existsSync(filepath)) { const stats = fs.statSync(filepath); console.log(`File size: ${stats.size} bytes`); } } else { const error = 'Download link not found after clicking Export'; console.log(error); await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'download-not-found.png') }); sendErrorEmail(error); throw new Error(error); } } else { const error = 'Export button not found on page'; console.log(error); sendErrorEmail(error); throw new Error(error); } console.log('Done!'); } catch (error) { console.error('Error:', error.message); await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'error.png') }); sendErrorEmail(error.message); throw error; } finally { await browser.close(); } } downloadRoster().catch(err => { console.error('Failed:', err); process.exit(1); });