Pulling data nightly

# One page of yesterday's brands by engine; YESTERDAY is yesterday in your market's time zone.
# Repeat with &cursor=$NEXT_CURSOR until has_more is false.
curl "https://api.citino.ai/v1/projects/$PROJECT_ID/reports/brands?from=$YESTERDAY&to=$YESTERDAY&dimensions=date,engine" -H "Authorization: Bearer $CITINO_API_KEY"

import os
import time
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
import requests

BASE = "https://api.citino.ai/v1"
key = os.environ["CITINO_API_KEY"]
project_id = os.environ["PROJECT_ID"]
session = requests.Session()
session.headers["Authorization"] = f"Bearer {key}"


def pull(report, day, dimensions="date,engine"):
    """Every row of one report for one day."""
    params = {"from": day, "to": day, "dimensions": dimensions}
    while True:
        response = session.get(f"{BASE}/projects/{project_id}/reports/{report}", params=params)
        if response.status_code in (429, 503) and response.json()["error"]["code"] in ("rate_limited", "busy"):
            time.sleep(int(response.headers.get("Retry-After", "5")))
            continue
        response.raise_for_status()
        page = response.json()
        yield from page["items"]
        if not page["has_more"]:
            return
        params["cursor"] = page["next_cursor"]


market = ZoneInfo(os.environ["MARKET_TZ"])  # e.g. America/New_York
yesterday = (datetime.now(market).date() - timedelta(days=1)).isoformat()
for row in pull("brands", yesterday):
    key_columns = (row["date"], row["engine"], row["brand"]["id"])
    v = row["visibility"]
    # Replace the stored row with this key: upsert(key_columns, row)
    print(key_columns, v["value"], v["band"], v["n"], v["low_sample"])

const BASE = "https://api.citino.ai/v1";
const headers = { Authorization: `Bearer ${process.env.CITINO_API_KEY}` };
const sleep = (seconds) => new Promise((r) => setTimeout(r, seconds * 1000));

// Every row of one report for one day.
async function* pull(report, day, dimensions = "date,engine") {
  const params = { from: day, to: day, dimensions };
  for (;;) {
    const query = new URLSearchParams(params);
    const response = await fetch(
      `${BASE}/projects/${process.env.PROJECT_ID}/reports/${report}?${query}`,
      { headers },
    );
    const body = await response.json();
    if (["rate_limited", "busy"].includes(body.error?.code)) {
      await sleep(Number(response.headers.get("Retry-After") ?? "5"));
      continue;
    }
    if (!response.ok) throw new Error(`${body.error.code}: ${body.error.message}`);
    yield* body.items;
    if (!body.has_more) return;
    params.cursor = body.next_cursor;
  }
}

// Yesterday in the market's time zone (MARKET_TZ, e.g. America/New_York).
const today = new Intl.DateTimeFormat("en-CA", { timeZone: process.env.MARKET_TZ }).format(new Date());
const day = new Date(`${today}T00:00:00Z`);
day.setUTCDate(day.getUTCDate() - 1);
const yesterday = day.toISOString().slice(0, 10);
for await (const row of pull("brands", yesterday)) {
  const key = [row.date, row.engine, row.brand.id];
  const { value, band, n, low_sample } = row.visibility;
  // Replace the stored row with this key: upsert(key, row)
  console.log(key, value, band, n, low_sample);
}