Skip to content

Instantly share code, notes, and snippets.

@cristianvasquez
Created March 5, 2026 15:49
Show Gist options
  • Select an option

  • Save cristianvasquez/e20af4890d5a97fa0084501a8fc7ece8 to your computer and use it in GitHub Desktop.

Select an option

Save cristianvasquez/e20af4890d5a97fa0084501a8fc7ece8 to your computer and use it in GitHub Desktop.
EP Roll-Call Vote Downloader: browser SPA (DuckDB WASM) + local proxy server
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>EP Roll-Call Vote Downloader</title>
<script src="https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser.js"></script>
<style>
*, *::before, *::after { box-sizing: border-box; }
body { font-family: system-ui, sans-serif; max-width: 820px; margin: 2rem auto; padding: 0 1.2rem; color: #1f2937; }
h1 { font-size: 1.4rem; margin-bottom: .25rem; }
.subtitle { color: #6b7280; font-size: .9rem; margin: 0 0 1.5rem; }
.config { display: flex; gap: .8rem; align-items: center; flex-wrap: wrap; }
label { font-size: .9rem; color: #374151; }
input[type=text] { padding: .4rem .6rem; border: 1px solid #d1d5db; border-radius: 5px; font-size: 1rem; width: 100px; }
button { padding: .5rem 1.3rem; background: #2563eb; color: #fff; border: none; border-radius: 5px; cursor: pointer; font-size: 1rem; font-weight: 500; }
button:disabled { background: #93c5fd; cursor: default; }
#progress { margin-top: 1.8rem; }
#summary { font-size: .95rem; color: #374151; margin-bottom: .8rem; min-height: 1.4rem; }
table { border-collapse: collapse; width: 100%; font-size: .875rem; }
th { text-align: left; padding: .45rem .6rem; border-bottom: 2px solid #e5e7eb; color: #6b7280; font-weight: 600; }
td { padding: .35rem .6rem; border-bottom: 1px solid #f3f4f6; }
.st { font-size: 1rem; }
.pending { color: #d1d5db; }
.loading { color: #f59e0b; }
.done { color: #16a34a; }
.skipped { color: #9ca3af; }
.err { color: #dc2626; font-size: .8rem; }
#downloadArea { margin-top: 2rem; padding: 1rem 1.2rem; background: #f0fdf4; border: 1px solid #86efac; border-radius: 7px; display: flex; align-items: center; gap: 1.2rem; flex-wrap: wrap; }
#downloadArea p { margin: 0; font-size: .95rem; }
#downloadLink { display: inline-block; padding: .45rem 1.1rem; background: #16a34a; color: #fff; text-decoration: none; border-radius: 5px; font-weight: 500; font-size: .95rem; }
#downloadLink:hover { background: #15803d; }
.error-box { margin-top: 1rem; padding: .75rem 1rem; background: #fef2f2; border: 1px solid #fca5a5; border-radius: 5px; font-size: .9rem; color: #dc2626; }
.note { font-size: .8rem; color: #9ca3af; margin-top: 1.5rem; }
</style>
</head>
<body>
<h1>EP Roll-Call Vote Downloader</h1>
<p class="subtitle">Fetches plenary roll-call vote decisions from the European Parliament Open Data API and exports a CSV via DuckDB.</p>
<div class="config">
<label>Year(s) <input type="text" id="years" value="2025" placeholder="e.g. 2025"></label>
<button id="startBtn">Download &amp; Generate CSV</button>
</div>
<div id="progress" hidden>
<p id="summary"></p>
<table>
<thead><tr><th>Meeting</th><th>Date</th><th>Status</th><th style="text-align:right">Rows</th></tr></thead>
<tbody id="meetingRows"></tbody>
</table>
</div>
<div id="downloadArea" hidden>
<p>Done — <strong id="totalRows"></strong> rows</p>
<a id="downloadLink" href="#">⬇ Download CSV</a>
</div>
<div id="errorBox" class="error-box" hidden></div>
<p class="note">Note: must be served over HTTP (e.g. <code>python -m http.server</code>) — not via file://. Requires internet access for CDN + EP API.</p>
<script>
// ── EP API ─────────────────────────────────────────────────────────────────
const BASE = '/api/v2'; // proxied via server.mjs → data.europarl.europa.eu
const meetingsUrl = y => `${BASE}/meetings?year=${y}&format=application%2Fld%2Bjson&offset=0&limit=1000`;
const decisionsUrl = id => `${BASE}/meetings/${id}/decisions?vote-method=ROLL_CALL_EV&format=application%2Fld%2Bjson&json-layout=framed-and-included&offset=0&limit=1000`;
// ── DOM ────────────────────────────────────────────────────────────────────
const startBtn = document.getElementById('startBtn');
const progressDiv = document.getElementById('progress');
const summaryEl = document.getElementById('summary');
const meetingTbody = document.getElementById('meetingRows');
const downloadArea = document.getElementById('downloadArea');
const totalRowsEl = document.getElementById('totalRows');
const downloadLink = document.getElementById('downloadLink');
const errorBox = document.getElementById('errorBox');
function showError(msg) { errorBox.textContent = msg; errorBox.hidden = false; }
function delay(ms) { return new Promise(r => setTimeout(r, ms)); }
// ── JSON parsing ───────────────────────────────────────────────────────────
function pickLabel(obj) {
if (!obj || typeof obj !== 'object') return String(obj ?? '');
return obj.en ?? obj.fr ?? obj.mul ?? Object.values(obj)[0] ?? '';
}
function parseDecisions(json) {
// Build MEP lookup from included[]: "identifier" → { label, group }
const meps = {};
for (const item of (json.included ?? [])) {
if (item.identifier != null) {
meps[String(item.identifier)] = {
label: item.label ?? '',
group: item['api:political-group'] ?? '',
};
}
}
const rows = [];
for (const vote of (json.data ?? [])) {
const activity_id = vote.activity_id ?? '';
const activity_date = vote.activity_date ?? '';
const activity_label = pickLabel(vote.activity_label);
for (const [prop, vote_type] of [
['had_voter_against', 'AGAINST'],
['had_voter_favor', 'FOR'],
['had_voter_abstention', 'ABSTENTION'],
]) {
for (const ref of (vote[prop] ?? [])) {
const id = String(ref).split('/').pop();
const mep = meps[id] ?? {};
rows.push([activity_id, activity_date, id, mep.label, mep.group, vote_type, activity_label]);
}
}
}
return rows;
}
// ── DuckDB ─────────────────────────────────────────────────────────────────
async function initDuckDB() {
const bundles = duckdb.getJsDelivrBundles();
const bundle = await duckdb.selectBundle(bundles);
const workerUrl = URL.createObjectURL(
new Blob([`importScripts("${bundle.mainWorker}");`], { type: 'text/javascript' })
);
const worker = new Worker(workerUrl);
const db = new duckdb.AsyncDuckDB(new duckdb.ConsoleLogger(), worker);
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
return db;
}
function sqlStr(s) {
return "'" + String(s ?? '').replace(/'/g, "''") + "'";
}
async function batchInsert(conn, rows) {
const BATCH = 2000;
for (let i = 0; i < rows.length; i += BATCH) {
const vals = rows.slice(i, i + BATCH)
.map(r => `(${r.map(sqlStr).join(',')})`)
.join(',');
await conn.query(`INSERT INTO votes VALUES ${vals}`);
}
}
// ── Main ───────────────────────────────────────────────────────────────────
startBtn.addEventListener('click', async () => {
startBtn.disabled = true;
errorBox.hidden = true;
downloadArea.hidden = true;
progressDiv.hidden = false;
meetingTbody.innerHTML = '';
const years = document.getElementById('years').value
.split(/[\s,]+/).map(s => s.trim()).filter(Boolean).map(Number)
.filter(n => !isNaN(n));
if (!years.length) {
showError('Enter at least one valid year.');
startBtn.disabled = false;
return;
}
summaryEl.textContent = 'Initialising DuckDB…';
let db, conn;
try {
db = await initDuckDB();
conn = await db.connect();
await conn.query(`
CREATE TABLE votes (
activity_id VARCHAR,
activity_date VARCHAR,
had_voter VARCHAR,
label VARCHAR,
political_group VARCHAR,
vote_type VARCHAR,
activity_label VARCHAR
)
`);
} catch (e) {
showError(`DuckDB init failed: ${e.message}`);
startBtn.disabled = false;
return;
}
let totalRows = 0;
for (const year of years) {
summaryEl.textContent = `Fetching meeting list for ${year}…`;
let meetings;
try {
const res = await fetch(meetingsUrl(year));
if (!res.ok) throw new Error(`HTTP ${res.status}`);
meetings = (await res.json()).data ?? [];
} catch (e) {
showError(`Failed to fetch meetings for ${year}: ${e.message}`);
startBtn.disabled = false;
return;
}
// Render all meetings as pending
for (const m of meetings) {
const tr = document.createElement('tr');
tr.innerHTML = `
<td>${m.activity_id}</td>
<td>${m.activity_date ?? ''}</td>
<td class="st pending" id="st-${m.activity_id}">⏳</td>
<td style="text-align:right" id="rc-${m.activity_id}">—</td>`;
meetingTbody.appendChild(tr);
}
for (let i = 0; i < meetings.length; i++) {
const m = meetings[i];
const stEl = document.getElementById(`st-${m.activity_id}`);
const rcEl = document.getElementById(`rc-${m.activity_id}`);
stEl.className = 'st loading';
stEl.textContent = '⬇';
summaryEl.textContent = `${year}: ${i + 1} / ${meetings.length} meetings — ${totalRows.toLocaleString()} rows`;
await delay(300);
let rows = [];
try {
const res = await fetch(decisionsUrl(m.activity_id));
if (res.status === 404) {
stEl.className = 'st skipped'; stEl.textContent = '—'; rcEl.textContent = '0';
continue;
}
if (!res.ok) throw new Error(`HTTP ${res.status}`);
rows = parseDecisions(await res.json());
} catch (e) {
stEl.className = 'st err'; stEl.textContent = '✗'; rcEl.textContent = e.message;
continue;
}
if (rows.length > 0) await batchInsert(conn, rows);
totalRows += rows.length;
stEl.className = 'st done'; stEl.textContent = '✓';
rcEl.textContent = rows.length.toLocaleString();
}
}
// Export CSV from DuckDB
summaryEl.textContent = `Exporting ${totalRows.toLocaleString()} rows…`;
try {
await conn.query(`COPY votes TO 'output.csv' (HEADER true, DELIMITER ',')`);
const buf = await db.copyFileToBuffer('output.csv');
const blob = new Blob([buf], { type: 'text/csv' });
downloadLink.href = URL.createObjectURL(blob);
downloadLink.download = `ep-votes-${years.join('-')}.csv`;
totalRowsEl.textContent = totalRows.toLocaleString();
downloadArea.hidden = false;
summaryEl.textContent = `Complete — ${meetings?.length ?? '?'} meetings, ${totalRows.toLocaleString()} rows.`;
} catch (e) {
showError(`CSV export failed: ${e.message}`);
}
startBtn.disabled = false;
});
</script>
</body>
</html>
// Local dev server: serves app.html and proxies /api/* → EP Open Data API
// Usage: node server.mjs (then open http://localhost:8080)
import http from 'http';
import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PORT = 8080;
const EP_BASE = 'https://data.europarl.europa.eu';
const server = http.createServer(async (req, res) => {
const url = new URL(req.url, `http://localhost:${PORT}`);
// Proxy /api/* to the EP API
if (url.pathname.startsWith('/api/')) {
const target = EP_BASE + url.pathname + url.search;
try {
const upstream = await fetch(target, {
headers: { 'User-Agent': 'ep-vote-spa/1.0', 'Accept': 'application/ld+json' },
});
const body = await upstream.arrayBuffer();
res.writeHead(upstream.status, {
'Content-Type': upstream.headers.get('content-type') ?? 'application/json',
'Access-Control-Allow-Origin': '*',
});
res.end(Buffer.from(body));
} catch (e) {
res.writeHead(502);
res.end(`Proxy error: ${e.message}`);
}
return;
}
// Serve app.html for /
if (url.pathname === '/' || url.pathname === '/app.html') {
try {
const html = await fs.readFile(path.join(__dirname, 'app.html'));
res.writeHead(200, { 'Content-Type': 'text/html' });
res.end(html);
} catch {
res.writeHead(404); res.end('Not found');
}
return;
}
res.writeHead(404); res.end('Not found');
});
server.listen(PORT, () => {
console.log(`Server running at http://localhost:${PORT}`);
});
@cristianvasquez
Copy link
Author

image

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment