Created
March 5, 2026 15:49
-
-
Save cristianvasquez/e20af4890d5a97fa0084501a8fc7ece8 to your computer and use it in GitHub Desktop.
EP Roll-Call Vote Downloader: browser SPA (DuckDB WASM) + local proxy server
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>EP Roll-Call Vote Downloader</title> | |
| <script src="https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser.js"></script> | |
| <style> | |
| *, *::before, *::after { box-sizing: border-box; } | |
| body { font-family: system-ui, sans-serif; max-width: 820px; margin: 2rem auto; padding: 0 1.2rem; color: #1f2937; } | |
| h1 { font-size: 1.4rem; margin-bottom: .25rem; } | |
| .subtitle { color: #6b7280; font-size: .9rem; margin: 0 0 1.5rem; } | |
| .config { display: flex; gap: .8rem; align-items: center; flex-wrap: wrap; } | |
| label { font-size: .9rem; color: #374151; } | |
| input[type=text] { padding: .4rem .6rem; border: 1px solid #d1d5db; border-radius: 5px; font-size: 1rem; width: 100px; } | |
| button { padding: .5rem 1.3rem; background: #2563eb; color: #fff; border: none; border-radius: 5px; cursor: pointer; font-size: 1rem; font-weight: 500; } | |
| button:disabled { background: #93c5fd; cursor: default; } | |
| #progress { margin-top: 1.8rem; } | |
| #summary { font-size: .95rem; color: #374151; margin-bottom: .8rem; min-height: 1.4rem; } | |
| table { border-collapse: collapse; width: 100%; font-size: .875rem; } | |
| th { text-align: left; padding: .45rem .6rem; border-bottom: 2px solid #e5e7eb; color: #6b7280; font-weight: 600; } | |
| td { padding: .35rem .6rem; border-bottom: 1px solid #f3f4f6; } | |
| .st { font-size: 1rem; } | |
| .pending { color: #d1d5db; } | |
| .loading { color: #f59e0b; } | |
| .done { color: #16a34a; } | |
| .skipped { color: #9ca3af; } | |
| .err { color: #dc2626; font-size: .8rem; } | |
| #downloadArea { margin-top: 2rem; padding: 1rem 1.2rem; background: #f0fdf4; border: 1px solid #86efac; border-radius: 7px; display: flex; align-items: center; gap: 1.2rem; flex-wrap: wrap; } | |
| #downloadArea p { margin: 0; font-size: .95rem; } | |
| #downloadLink { display: inline-block; padding: .45rem 1.1rem; background: #16a34a; color: #fff; text-decoration: none; border-radius: 5px; font-weight: 500; font-size: .95rem; } | |
| #downloadLink:hover { background: #15803d; } | |
| .error-box { margin-top: 1rem; padding: .75rem 1rem; background: #fef2f2; border: 1px solid #fca5a5; border-radius: 5px; font-size: .9rem; color: #dc2626; } | |
| .note { font-size: .8rem; color: #9ca3af; margin-top: 1.5rem; } | |
| </style> | |
| </head> | |
| <body> | |
| <h1>EP Roll-Call Vote Downloader</h1> | |
| <p class="subtitle">Fetches plenary roll-call vote decisions from the European Parliament Open Data API and exports a CSV via DuckDB.</p> | |
| <div class="config"> | |
| <label>Year(s) <input type="text" id="years" value="2025" placeholder="e.g. 2025"></label> | |
| <button id="startBtn">Download & Generate CSV</button> | |
| </div> | |
| <div id="progress" hidden> | |
| <p id="summary"></p> | |
| <table> | |
| <thead><tr><th>Meeting</th><th>Date</th><th>Status</th><th style="text-align:right">Rows</th></tr></thead> | |
| <tbody id="meetingRows"></tbody> | |
| </table> | |
| </div> | |
| <div id="downloadArea" hidden> | |
| <p>Done — <strong id="totalRows"></strong> rows</p> | |
| <a id="downloadLink" href="#">⬇ Download CSV</a> | |
| </div> | |
| <div id="errorBox" class="error-box" hidden></div> | |
| <p class="note">Note: must be served over HTTP (e.g. <code>python -m http.server</code>) — not via file://. Requires internet access for CDN + EP API.</p> | |
| <script> | |
| // ── EP API ───────────────────────────────────────────────────────────────── | |
| const BASE = '/api/v2'; // proxied via server.mjs → data.europarl.europa.eu | |
| const meetingsUrl = y => `${BASE}/meetings?year=${y}&format=application%2Fld%2Bjson&offset=0&limit=1000`; | |
| const decisionsUrl = id => `${BASE}/meetings/${id}/decisions?vote-method=ROLL_CALL_EV&format=application%2Fld%2Bjson&json-layout=framed-and-included&offset=0&limit=1000`; | |
| // ── DOM ──────────────────────────────────────────────────────────────────── | |
| const startBtn = document.getElementById('startBtn'); | |
| const progressDiv = document.getElementById('progress'); | |
| const summaryEl = document.getElementById('summary'); | |
| const meetingTbody = document.getElementById('meetingRows'); | |
| const downloadArea = document.getElementById('downloadArea'); | |
| const totalRowsEl = document.getElementById('totalRows'); | |
| const downloadLink = document.getElementById('downloadLink'); | |
| const errorBox = document.getElementById('errorBox'); | |
| function showError(msg) { errorBox.textContent = msg; errorBox.hidden = false; } | |
| function delay(ms) { return new Promise(r => setTimeout(r, ms)); } | |
| // ── JSON parsing ─────────────────────────────────────────────────────────── | |
| function pickLabel(obj) { | |
| if (!obj || typeof obj !== 'object') return String(obj ?? ''); | |
| return obj.en ?? obj.fr ?? obj.mul ?? Object.values(obj)[0] ?? ''; | |
| } | |
| function parseDecisions(json) { | |
| // Build MEP lookup from included[]: "identifier" → { label, group } | |
| const meps = {}; | |
| for (const item of (json.included ?? [])) { | |
| if (item.identifier != null) { | |
| meps[String(item.identifier)] = { | |
| label: item.label ?? '', | |
| group: item['api:political-group'] ?? '', | |
| }; | |
| } | |
| } | |
| const rows = []; | |
| for (const vote of (json.data ?? [])) { | |
| const activity_id = vote.activity_id ?? ''; | |
| const activity_date = vote.activity_date ?? ''; | |
| const activity_label = pickLabel(vote.activity_label); | |
| for (const [prop, vote_type] of [ | |
| ['had_voter_against', 'AGAINST'], | |
| ['had_voter_favor', 'FOR'], | |
| ['had_voter_abstention', 'ABSTENTION'], | |
| ]) { | |
| for (const ref of (vote[prop] ?? [])) { | |
| const id = String(ref).split('/').pop(); | |
| const mep = meps[id] ?? {}; | |
| rows.push([activity_id, activity_date, id, mep.label, mep.group, vote_type, activity_label]); | |
| } | |
| } | |
| } | |
| return rows; | |
| } | |
| // ── DuckDB ───────────────────────────────────────────────────────────────── | |
| async function initDuckDB() { | |
| const bundles = duckdb.getJsDelivrBundles(); | |
| const bundle = await duckdb.selectBundle(bundles); | |
| const workerUrl = URL.createObjectURL( | |
| new Blob([`importScripts("${bundle.mainWorker}");`], { type: 'text/javascript' }) | |
| ); | |
| const worker = new Worker(workerUrl); | |
| const db = new duckdb.AsyncDuckDB(new duckdb.ConsoleLogger(), worker); | |
| await db.instantiate(bundle.mainModule, bundle.pthreadWorker); | |
| return db; | |
| } | |
| function sqlStr(s) { | |
| return "'" + String(s ?? '').replace(/'/g, "''") + "'"; | |
| } | |
| async function batchInsert(conn, rows) { | |
| const BATCH = 2000; | |
| for (let i = 0; i < rows.length; i += BATCH) { | |
| const vals = rows.slice(i, i + BATCH) | |
| .map(r => `(${r.map(sqlStr).join(',')})`) | |
| .join(','); | |
| await conn.query(`INSERT INTO votes VALUES ${vals}`); | |
| } | |
| } | |
| // ── Main ─────────────────────────────────────────────────────────────────── | |
| startBtn.addEventListener('click', async () => { | |
| startBtn.disabled = true; | |
| errorBox.hidden = true; | |
| downloadArea.hidden = true; | |
| progressDiv.hidden = false; | |
| meetingTbody.innerHTML = ''; | |
| const years = document.getElementById('years').value | |
| .split(/[\s,]+/).map(s => s.trim()).filter(Boolean).map(Number) | |
| .filter(n => !isNaN(n)); | |
| if (!years.length) { | |
| showError('Enter at least one valid year.'); | |
| startBtn.disabled = false; | |
| return; | |
| } | |
| summaryEl.textContent = 'Initialising DuckDB…'; | |
| let db, conn; | |
| try { | |
| db = await initDuckDB(); | |
| conn = await db.connect(); | |
| await conn.query(` | |
| CREATE TABLE votes ( | |
| activity_id VARCHAR, | |
| activity_date VARCHAR, | |
| had_voter VARCHAR, | |
| label VARCHAR, | |
| political_group VARCHAR, | |
| vote_type VARCHAR, | |
| activity_label VARCHAR | |
| ) | |
| `); | |
| } catch (e) { | |
| showError(`DuckDB init failed: ${e.message}`); | |
| startBtn.disabled = false; | |
| return; | |
| } | |
| let totalRows = 0; | |
| for (const year of years) { | |
| summaryEl.textContent = `Fetching meeting list for ${year}…`; | |
| let meetings; | |
| try { | |
| const res = await fetch(meetingsUrl(year)); | |
| if (!res.ok) throw new Error(`HTTP ${res.status}`); | |
| meetings = (await res.json()).data ?? []; | |
| } catch (e) { | |
| showError(`Failed to fetch meetings for ${year}: ${e.message}`); | |
| startBtn.disabled = false; | |
| return; | |
| } | |
| // Render all meetings as pending | |
| for (const m of meetings) { | |
| const tr = document.createElement('tr'); | |
| tr.innerHTML = ` | |
| <td>${m.activity_id}</td> | |
| <td>${m.activity_date ?? ''}</td> | |
| <td class="st pending" id="st-${m.activity_id}">⏳</td> | |
| <td style="text-align:right" id="rc-${m.activity_id}">—</td>`; | |
| meetingTbody.appendChild(tr); | |
| } | |
| for (let i = 0; i < meetings.length; i++) { | |
| const m = meetings[i]; | |
| const stEl = document.getElementById(`st-${m.activity_id}`); | |
| const rcEl = document.getElementById(`rc-${m.activity_id}`); | |
| stEl.className = 'st loading'; | |
| stEl.textContent = '⬇'; | |
| summaryEl.textContent = `${year}: ${i + 1} / ${meetings.length} meetings — ${totalRows.toLocaleString()} rows`; | |
| await delay(300); | |
| let rows = []; | |
| try { | |
| const res = await fetch(decisionsUrl(m.activity_id)); | |
| if (res.status === 404) { | |
| stEl.className = 'st skipped'; stEl.textContent = '—'; rcEl.textContent = '0'; | |
| continue; | |
| } | |
| if (!res.ok) throw new Error(`HTTP ${res.status}`); | |
| rows = parseDecisions(await res.json()); | |
| } catch (e) { | |
| stEl.className = 'st err'; stEl.textContent = '✗'; rcEl.textContent = e.message; | |
| continue; | |
| } | |
| if (rows.length > 0) await batchInsert(conn, rows); | |
| totalRows += rows.length; | |
| stEl.className = 'st done'; stEl.textContent = '✓'; | |
| rcEl.textContent = rows.length.toLocaleString(); | |
| } | |
| } | |
| // Export CSV from DuckDB | |
| summaryEl.textContent = `Exporting ${totalRows.toLocaleString()} rows…`; | |
| try { | |
| await conn.query(`COPY votes TO 'output.csv' (HEADER true, DELIMITER ',')`); | |
| const buf = await db.copyFileToBuffer('output.csv'); | |
| const blob = new Blob([buf], { type: 'text/csv' }); | |
| downloadLink.href = URL.createObjectURL(blob); | |
| downloadLink.download = `ep-votes-${years.join('-')}.csv`; | |
| totalRowsEl.textContent = totalRows.toLocaleString(); | |
| downloadArea.hidden = false; | |
| summaryEl.textContent = `Complete — ${meetings?.length ?? '?'} meetings, ${totalRows.toLocaleString()} rows.`; | |
| } catch (e) { | |
| showError(`CSV export failed: ${e.message}`); | |
| } | |
| startBtn.disabled = false; | |
| }); | |
| </script> | |
| </body> | |
| </html> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Local dev server: serves app.html and proxies /api/* → EP Open Data API | |
| // Usage: node server.mjs (then open http://localhost:8080) | |
| import http from 'http'; | |
| import fs from 'fs/promises'; | |
| import path from 'path'; | |
| import { fileURLToPath } from 'url'; | |
| const __dirname = path.dirname(fileURLToPath(import.meta.url)); | |
| const PORT = 8080; | |
| const EP_BASE = 'https://data.europarl.europa.eu'; | |
| const server = http.createServer(async (req, res) => { | |
| const url = new URL(req.url, `http://localhost:${PORT}`); | |
| // Proxy /api/* to the EP API | |
| if (url.pathname.startsWith('/api/')) { | |
| const target = EP_BASE + url.pathname + url.search; | |
| try { | |
| const upstream = await fetch(target, { | |
| headers: { 'User-Agent': 'ep-vote-spa/1.0', 'Accept': 'application/ld+json' }, | |
| }); | |
| const body = await upstream.arrayBuffer(); | |
| res.writeHead(upstream.status, { | |
| 'Content-Type': upstream.headers.get('content-type') ?? 'application/json', | |
| 'Access-Control-Allow-Origin': '*', | |
| }); | |
| res.end(Buffer.from(body)); | |
| } catch (e) { | |
| res.writeHead(502); | |
| res.end(`Proxy error: ${e.message}`); | |
| } | |
| return; | |
| } | |
| // Serve app.html for / | |
| if (url.pathname === '/' || url.pathname === '/app.html') { | |
| try { | |
| const html = await fs.readFile(path.join(__dirname, 'app.html')); | |
| res.writeHead(200, { 'Content-Type': 'text/html' }); | |
| res.end(html); | |
| } catch { | |
| res.writeHead(404); res.end('Not found'); | |
| } | |
| return; | |
| } | |
| res.writeHead(404); res.end('Not found'); | |
| }); | |
| server.listen(PORT, () => { | |
| console.log(`Server running at http://localhost:${PORT}`); | |
| }); |
Author
cristianvasquez
commented
Mar 5, 2026
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment