wifi-etl/backfill.py

import time
from datetime import date, timedelta
import psycopg2

from app.extractor.wifire import extract_customers
from app.core.config import DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD
from app.transform.merge_mac import transform_wifire
from app.load.load_database import load

START_DATE = date(2026, 4, 21)
END_DATE   = date.today() - timedelta(days=1)

SLEEP_BETWEEN = 20  # 20s entre requisições = 3 req/min, abaixo do limite de 10 req/min

conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD)
conn.autocommit = False

current = START_DATE
while current <= END_DATE:
    print(f"Processando {current}...")
    try:
        raw, _ = extract_customers(watermark_date=current)
        users = [transform_wifire(r) for r in raw]
        if users:
            load(conn, users, [])
            conn.commit()
            print(f"  {len(users)} usuários inseridos")
        else:
            print(f"  sem dados")
    except Exception as e:
        conn.rollback()
        if "429" in str(e):
            print(f"  Rate limit — aguardando 90s...")
            time.sleep(90)
        else:
            print(f"  ERRO em {current}: {e}")

    current += timedelta(days=1)
    time.sleep(SLEEP_BETWEEN)

conn.close()
print("Backfill concluído!")