Pagination

const { ACTIVITY_URL, ACCOUNT, LUMEUS_API_KEY } = process.env;
const headers = { Authorization: `Bearer ${LUMEUS_API_KEY}` };
const head = await fetch(`${ACTIVITY_URL}/accounts/${ACCOUNT}/activity?limit=1`, { headers });
const first = await head.json().catch(() => ({}));
if (!head.ok) throw new Error(`${head.status} ${first.code ?? head.statusText}: ${first.error} (${first.request_id})`);
let next = `/accounts/${ACCOUNT}/activity?order=asc&limit=1000&since_ledger=${first.read_view.published_through - 200}`;

const RETRYABLE = ['rate_limited', 'busy', 'index_not_ready', 'unavailable'];

for (let pages = 0, retries = 0; next && pages < 50; ) {
  const response = await fetch(new URL(next, ACTIVITY_URL), { headers });
  const page = await response.json().catch(() => ({})); // an edge 502 can be HTML
  const wait = Number(response.headers.get('retry-after') ?? 2);
  if (!response.ok && RETRYABLE.includes(page.code) && retries < 5 && wait <= 60) {
    retries++;
    await new Promise(r => setTimeout(r, wait * 1000));
    continue; // same URL again
  }
  if (!response.ok) throw new Error(`${response.status} ${page.code}: ${page.error} (${page.request_id})`);
  pages++;
  retries = 0;
  console.log(page._embedded.records.length, 'records');
  next = page.range_complete ? null : page._links.next.href;
  if (!next) console.log('checkpoint', page.scanned_through_ledger);
}

import os
import time
import requests

base, account = os.environ["ACTIVITY_URL"], os.environ["ACCOUNT"]
headers = {"Authorization": f"Bearer {os.environ['LUMEUS_API_KEY']}"}
head = requests.get(f"{base}/accounts/{account}/activity?limit=1", headers=headers, timeout=60)
if not head.ok:
    try:
        err = head.json()
    except ValueError:  # an edge 502 can be HTML
        err = {}
    raise RuntimeError(f"{head.status_code} {err.get('code', head.reason)}: {err.get('error')} ({err.get('request_id')})")
first = head.json()
next_url = f"/accounts/{account}/activity?order=asc&limit=1000&since_ledger={first['read_view']['published_through'] - 200}"

RETRYABLE = {"rate_limited", "busy", "index_not_ready", "unavailable"}
pages = retries = 0
while next_url and pages < 50:
    response = requests.get(base + next_url, headers=headers, timeout=60)
    try:
        page = response.json()
    except ValueError:  # an edge 502 can be HTML
        page = {}
    wait = int(response.headers.get("Retry-After", "2"))
    if not response.ok and page.get("code") in RETRYABLE and retries < 5 and wait <= 60:
        retries += 1
        time.sleep(wait)
        continue  # same URL again
    if not response.ok:
        raise RuntimeError(f"{response.status_code} {page.get('code')}: {page.get('error')} ({page.get('request_id')})")
    pages, retries = pages + 1, 0
    print(len(page["_embedded"]["records"]), "records")
    next_url = None if page["range_complete"] else page["_links"]["next"]["href"]
    if not next_url:
        print("checkpoint", page["scanned_through_ledger"])