lwvwv/download-roster.js

#!/usr/bin/env node
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const DOWNLOAD_DIR = path.dirname(__filename);
const SESSION_FILE = path.join(DOWNLOAD_DIR, '.session.json');
const ENV_FILE = path.join(DOWNLOAD_DIR, 'env');

// Read configuration from env file
function loadConfig() {
  const content = fs.readFileSync(ENV_FILE, 'utf8');
  
  const config = {};
  
  // Match $VAR = "value" patterns
  const matches = content.matchAll(/\$(\w+)\s*=\s*"([^"]*)"/g);
  for (const match of matches) {
    config[match[1]] = match[2];
  }
  
  return config;
}

const config = loadConfig();

const PORTAL_URL = config.PORTAL_URL || 'https://portal.lwv.org';
const MEMBERSHIP_URL = config.MEMBERSHIP_URL || 'https://portal.lwv.org/groups/43a93df1-901a-4676-88c3-f4ea430d4884/league_membership_state_view';

// Email config
const SMTP_HOST = config.SMTP_HOST || '';
const SMTP_PORT = config.SMTP_PORT || '587';
const SMTP_USE_STARTTLS = config.SMTP_USE_STARTTLS || 'true';
const SMTP_AUTH = config.SMTP_AUTH || 'login';
const SMTP_USER = config.SMTP_USER || '';
const SMTP_PASSWORD = config.SMTP_PASSWORD || '';
const SSL_VERIFY_IGNORE = config.SSL_VERIFY_IGNORE || 'ignore';
const EMAIL_FROM = config.EMAIL_FROM || '';
const EMAIL_TO = config.EMAIL_TO || '';

function loadSession() {
  if (!fs.existsSync(SESSION_FILE)) {
    return null;
  }
  
  try {
    const sessionData = JSON.parse(fs.readFileSync(SESSION_FILE, 'utf8'));
    
    // Check if expired
    if (new Date(sessionData.expiresAt) < new Date()) {
      console.log('Session expired');
      return null;
    }
    
    return sessionData;
  } catch (e) {
    console.error('Error loading session:', e.message);
    return null;
  }
}

function sendErrorEmail(errorMessage) {
  if (!SMTP_HOST || !EMAIL_TO) {
    console.log('Email not configured, skipping notification');
    return;
  }
  
  const hostname = require('os').hostname();
  const subject = `LWV Roster Download Failed on: ${hostname}`;
  const body = `Error: ${errorMessage}\n\nTo fix this, please run: node save-session.js\n\nThen log in using the magic link method to refresh your session.`;
  
  // Build s-nail command - match user's working format exactly
  // Note: smtp value needs quotes because it contains ":"
  const snaicmd = `s-nail -r "${EMAIL_FROM}" -s "${subject}" -S smtp="${SMTP_HOST}:${SMTP_PORT}" -S smtp-use-starttls -S smtp-auth=${SMTP_AUTH} -S smtp-auth-user="${SMTP_USER}" -S smtp-auth-password="${SMTP_PASSWORD}" -S ssl-verify=${SSL_VERIFY_IGNORE} ${EMAIL_TO}`;
  
  // Write body to temp file to avoid shell escaping issues
  const tempFile = '/tmp/lwv-email-body.txt';
  fs.writeFileSync(tempFile, body);
  
  try {
    execSync(`cat ${tempFile} | ${snaicmd}`, { stdio: 'pipe' }); 
    console.log('Error notification email sent');
    fs.unlinkSync(tempFile);
  } catch (e) {
    console.log('Failed to send error email:', e.message);
    try { fs.unlinkSync(tempFile); } catch {}
  }
}

async function downloadRoster() {
  // Try to load saved session
  const sessionData = loadSession();
  
  console.log('Starting browser...');
  
  const browser = await chromium.launch({
    headless: true,
    downloadsDir: DOWNLOAD_DIR
  });
  
  let context;
  let page;
  
  // If we have a valid session, use it
  if (sessionData) {
    console.log('Using saved session from', sessionData.savedAt);
    
    // Use storageState if available (includes cookies and localStorage)
    if (sessionData.storageState) {
      console.log('Using storageState to restore session...');
      context = await browser.newContext({
        storageState: sessionData.storageState
      });
    } else {
      context = await browser.newContext();
      await context.addCookies(sessionData.cookies);
    }
  } else {
    const error = 'No valid session found. Run save-session.js first to create one.';
    console.log(error);
    sendErrorEmail(error);
    await browser.close();
    process.exit(1);
  }
  
  // Debug: check what cookies were added
  const addedCookies = await context.cookies();
  console.log('Browser has', addedCookies.length, 'cookies');
  
  page = await context.newPage();
  
  try {
    // Navigate to portal home first to establish session
    console.log('Navigating to portal home...');
    await page.goto(PORTAL_URL, { waitUntil: 'networkidle', timeout: 60000 });
    await page.waitForTimeout(2000);
    console.log('Home URL:', page.url());
    
    // Check if still on login page
    const homePageText = await page.textContent('body').catch(() => '');
    console.log('Home page contains login:', homePageText.includes('Continue with email'));
    
    // If still on login page, session is definitely invalid
    if (homePageText.includes('Continue with email')) {
      const error = 'Session has expired - redirected to login page. Run save-session.js to refresh.';
      console.log(error);
      sendErrorEmail(error);
      await browser.close();
      process.exit(1);
    }
    
    // Now navigate to membership page
    console.log('Navigating to membership page...');
    await page.goto(MEMBERSHIP_URL, { waitUntil: 'networkidle', timeout: 60000 });
    await page.waitForTimeout(3000);
    
    console.log('URL:', page.url());
    
    // Debug: print page title
    console.log('Page title:', await page.title());
    
    // Debug: check if we're on login page
    const pageContent = await page.content();
    const pageText = await page.textContent('body').catch(() => '');
    
    console.log('Page text length:', pageText.length);
    console.log('Contains Continue with email:', pageText.includes('Continue with email'));
    
    // Verify we're on the right page - check for login page elements
    const isLoginPage = pageText.includes('Continue with email') || 
                        pageText.includes('Continue with password') ||
                        pageText.includes('Log in');
    
    if (isLoginPage) {
      const error = 'Session has expired - redirected to login page. Run save-session.js to refresh.';
      console.log(error);
      sendErrorEmail(error);
      await browser.close();
      process.exit(1);
    }
    
    // Look for Export button
    console.log('Looking for Export button...');
    await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'membership-page.png') });
    
    // Try to find Export button/link
    const exportBtn = await page.locator('button:has-text("Export")').first();
    
    if (await exportBtn.count() > 0) {
      console.log('Clicking Export...');
      await exportBtn.click();
      await page.waitForTimeout(3000);
      await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'after-export-click.png') });
      
      // Look for download button
      const downloadBtn = await page.locator('input[type="submit"][value="Download"]').first();
      if (await downloadBtn.count() > 0) {
        console.log('Clicking Download...');
        
        const downloadPromise = page.waitForEvent('download', { timeout: 30000 });
        await downloadBtn.click();
        
        const download = await downloadPromise;
        const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
        const filename = `league_membership_state_view_${timestamp}.csv`;
        const filepath = path.join(DOWNLOAD_DIR, filename);
        
        await download.saveAs(filepath);
        console.log(`Downloaded: ${filename}`);
        
        // Verify the file
        if (fs.existsSync(filepath)) {
          const stats = fs.statSync(filepath);
          console.log(`File size: ${stats.size} bytes`);
        }
      } else {
        const error = 'Download button not found in export modal';
        console.log(error);
        sendErrorEmail(error);
        throw new Error(error);
      }
    } else {
      const error = 'Export button not found on page';
      console.log(error);
      sendErrorEmail(error);
      throw new Error(error);
    }
    
    console.log('Done!');
    
  } catch (error) {
    console.error('Error:', error.message);
    await page.screenshot({ path: path.join(DOWNLOAD_DIR, 'error.png') });
    sendErrorEmail(error.message);
    throw error;
  } finally {
    await browser.close();
  }
}

downloadRoster().catch(err => {
  console.error('Failed:', err);
  process.exit(1);
});