import time from datetime import date, timedelta import psycopg2 from app.extractor.wifeed import get_access_token, extract_all_access from app.core.config import WIFEED_CLIENT_ID, WIFEED_CLIENT_SECRET from app.core.config import DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD from app.transform.merge_mac import transform_wifeed from app.load.load_database import load START_DATE = date(2026, 4, 21) END_DATE = date.today() - timedelta(days=1) SLEEP_BETWEEN = 20 # 20s entre requisições = 3 req/min, bem abaixo do limite conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD) conn.autocommit = False token = get_access_token(WIFEED_CLIENT_ID, WIFEED_CLIENT_SECRET) time.sleep(SLEEP_BETWEEN) # pausa já após o login current = START_DATE while current <= END_DATE: print(f"Processando {current}...") try: raw, _ = extract_all_access(token, watermark_date=current) users = [transform_wifeed(r) for r in raw if transform_wifeed(r)] if users: load(conn, users, []) # sessions vazio no backfill conn.commit() print(f" {len(users)} usuários inseridos") else: print(f" sem dados") except Exception as e: conn.rollback() if "429" in str(e): print(f" Rate limit — aguardando 90s...") time.sleep(90) else: print(f" ERRO em {current}: {e}") current += timedelta(days=1) time.sleep(SLEEP_BETWEEN) conn.close() print("Backfill concluído!")