Build your call log

"""call_log.py: copy your Riff calls into a folder you keep.

It makes a folder (call-log, or the name you give) with:
  calls.csv              one row per call, newest first. It opens in Excel and Google Sheets.
  transcripts/<id>.txt   what was said on each call, one line per turn ("Riff: ..." or "Caller: ...").
  .state.json            what the script already has, so the next run fetches only new and changed calls.

Put your key alone in a file, riff-key.txt in your home folder, then run it.
  macOS and Linux:       RIFF_API_KEY="$(cat ~/riff-key.txt)" python3 call_log.py
  Windows (PowerShell):  $env:RIFF_API_KEY = Get-Content ~/riff-key.txt
                         py call_log.py

It needs Python 3.9 or newer and nothing else. Riff keeps a call for 30 days; this folder keeps it for as long as
you like. Guide: https://whappai.com/developers-call-log.html
"""
import csv
import http.client
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timedelta, timezone

BASE = os.environ.get("RIFF_API_BASE", "https://api.chattychapters.com/api/v1").rstrip("/")
KEY = os.environ.get("RIFF_API_KEY", "").strip()
# The script names itself, so its requests are easy to pick out in logs (and a filter that refuses
# Python's default "Python-urllib/3.x" agent, 403 error 1010, lets it through).
USER_AGENT = "riff-call-log/1.0 (+https://whappai.com/developers-call-log.html)"
TIMEOUT = 30                                   # seconds to wait for one answer
RETRIES = 5                                    # how many times to wait and try again when the API is busy
REFETCH_ON = ("call.ended", "call.claimed")    # events that mean "this call is new or changed"
COLUMNS = ["id", "started_at", "duration_seconds", "kind", "line", "caller_name", "caller_phone", "need", "outcome",
           "booking_start", "booking_end", "address", "urgency", "transcript_file"]


class ApiError(Exception):
    """The API said no. `code` is the error code from the answer, like invalid_key."""


def get(path, params):
    """GET one API path and return its JSON answer. On 429 or 503, wait Retry-After seconds and try again."""
    query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})   # skip empty parameters
    url = BASE + path + ("?" + query if query else "")
    # The key goes in this header only. Never put it in the URL: URLs end up in logs.
    headers = {"Authorization": "Bearer " + KEY, "Accept": "application/json", "User-Agent": USER_AGENT}
    request = urllib.request.Request(url, headers=headers)
    for attempt in range(RETRIES + 1):
        try:
            with urllib.request.urlopen(request, timeout=TIMEOUT) as answer:
                return json.load(answer)
        except urllib.error.HTTPError as err:
            try:
                body = json.loads(err.read()).get("error", {})    # {"code": ..., "message": ...}
            except ValueError:                                   # not the API's JSON (a proxy's error page)
                body = {}
            if err.code in (429, 503) and attempt < RETRIES:
                retry_after = err.headers.get("Retry-After", "2")
                wait = int(retry_after) if retry_after.isdigit() else 2
                print(f"The API is busy ({err.code} {body.get('code')}). Waiting {wait} seconds.")
                time.sleep(wait)
                continue
            error = ApiError(f"{err.code} {body.get('code')}: {body.get('message')} "
                             f"(request id {err.headers.get('X-Request-Id')})")
            error.code = body.get("code")
            raise error


def walk_feed(after):
    """Read the event feed to its end. Return the new cursor, the ids of calls that ended or were claimed, and
    how many turns each ended call's transcript has (the call.ended event says so)."""
    ids, turns = set(), {}
    while True:
        page = get("/events", {"limit": 100, "after": after})
        ids.update(e["call_id"] for e in page["data"] if e["type"] in REFETCH_ON)
        turns.update((e["call_id"], e["data"]["turns"]) for e in page["data"] if e["type"] == "call.ended")
        after = page["next_cursor"] or after
        if not page["data"]:
            return after, ids, turns


def list_all_calls():
    """Every call of the last 30 days with its transcript, 20 per request (the most expand=transcript allows)."""
    calls, cursor = [], None
    while True:
        page = get("/calls", {"expand": "transcript", "limit": 20, "cursor": cursor})
        calls += page["data"]
        cursor = page["next_cursor"]
        if not cursor:
            return calls


def get_call(call_id):
    """One call with its transcript, or None when the API no longer has it (it passed 30 days)."""
    try:
        return get("/calls/" + urllib.parse.quote(call_id, safe=""), {"expand": "transcript"})
    except ApiError as err:
        if err.code == "not_found":
            return None
        raise


def is_complete(call, expected):
    """True when the transcript is in. It can be empty for a few seconds after a call ends, so an empty one is
    tried again next run. `expected` is the turn count from the call's call.ended event: 0 means nobody spoke.
    When the feed did not say (None), an empty transcript counts as final an hour after the call."""
    if call["transcript"]["turns"] or expected == 0:
        return True
    ended = datetime.fromisoformat(call["ended_at"].replace("Z", "+00:00"))
    return expected is None and datetime.now(timezone.utc) - ended > timedelta(hours=1)


def display_phone(phone):
    """+15125550147 becomes +1 512-555-0147. Spreadsheets keep this as text; a bare +1512... can turn into a number."""
    if phone and len(phone) == 12 and phone.startswith("+1") and phone[1:].isdigit():
        return f"+1 {phone[2:5]}-{phone[5:8]}-{phone[8:]}"
    return phone or ""


def file_name(call_id):
    """A safe file name for a call id (letters, digits, - and _ only)."""
    return "".join(c if c.isalnum() or c in "-_" else "_" for c in call_id)


def load_state(folder):
    """What earlier runs saved, or an empty state on the first run."""
    try:
        with open(os.path.join(folder, ".state.json"), encoding="utf-8") as f:
            state = json.load(f)
    except FileNotFoundError:
        return {"cursor": None, "calls": {}, "complete": {}, "turns": {}}
    except ValueError:
        sys.exit(f"{os.path.join(folder, '.state.json')} is damaged. Rename it and run again: "
                 "the script then copies every call again, and your other files stay.")
    state.setdefault("turns", {})
    return state


def save_state(folder, state):
    """Write the state to a temporary file, then swap it in, so a crash never leaves half a file."""
    path = os.path.join(folder, ".state.json")
    with open(path + ".tmp", "w", encoding="utf-8") as f:
        json.dump(state, f, ensure_ascii=False)
    os.replace(path + ".tmp", path)


def write_transcript(folder, call):
    """Write transcripts/<id>.txt and return its path inside the folder, or "" when nobody spoke yet."""
    turns = call.get("transcript", {}).get("turns", [])
    if not turns:
        return ""
    name = "transcripts/" + file_name(call["id"]) + ".txt"
    with open(os.path.join(folder, name), "w", encoding="utf-8") as f:
        f.writelines(("Riff" if t["speaker"] == "agent" else "Caller") + ": " + t["text"] + "\n" for t in turns)
    return name


def write_csv(folder, calls):
    """calls.csv, newest first. UTF-8 with a BOM and every field in quotes, so Excel reads it correctly."""
    rows = sorted(calls, key=lambda c: (c["started_at"] or c["ended_at"], c["id"]), reverse=True)
    with open(os.path.join(folder, "calls.csv.tmp"), "w", encoding="utf-8-sig", newline="") as f:
        out = csv.writer(f, quoting=csv.QUOTE_ALL)
        out.writerow(COLUMNS)
        for c in rows:
            booking, details = c["booking"] or {}, c["details"]
            out.writerow([c["id"], c["started_at"], c["duration_seconds"], c["kind"], display_phone(c["line"]),
                          c["caller"]["name"], display_phone(c["caller"]["phone"]), c["need"], c["outcome"],
                          booking.get("start"), booking.get("end"), details.get("address"), details.get("urgency"),
                          c.get("transcript_file", "")])
    os.replace(os.path.join(folder, "calls.csv.tmp"), os.path.join(folder, "calls.csv"))


def fetch(state):
    """Ask the API for what is new. Return the new feed cursor and the calls to store."""
    if state["cursor"] is None:
        print("First run: copying every call from the last 30 days.")
        # Take the feed cursor FIRST. A call that ends while we page through the list is then in the feed
        # after this cursor, and the next run fetches it. Nothing falls in the gap.
        cursor, _, turns = walk_feed(None)
        state["turns"].update(turns)
        return cursor, list_all_calls()
    print("Checking for new and changed calls.")
    cursor, ids, turns = walk_feed(state["cursor"])
    state["turns"].update(turns)
    ids.update(i for i, done in state["complete"].items() if not done)   # transcripts not in yet
    fetched = []
    for call_id in sorted(ids):
        call = get_call(call_id)
        if call is None:
            print(f"gone     {call_id} is past 30 days; your copy stays in the folder.")
            state["complete"][call_id] = True
        else:
            fetched.append(call)
    return cursor, fetched


def main():
    """Fetch what is new, update the folder, save where we stopped."""
    # A name the console cannot show (Windows, output to a file) prints as "?" instead of stopping the script.
    getattr(sys.stdout, "reconfigure", lambda **k: None)(errors="replace")
    folder = sys.argv[1] if len(sys.argv) > 1 else "call-log"
    if not KEY:
        sys.exit("Set RIFF_API_KEY to your key first. See Step 2 of the guide.")
    os.makedirs(os.path.join(folder, "transcripts"), exist_ok=True)
    state = load_state(folder)
    try:
        cursor, fetched = fetch(state)
    except ApiError as err:
        hint = " Check RIFF_API_KEY." if err.code == "invalid_key" else ""
        sys.exit(f"The API refused the request: {err}.{hint}")
    except (OSError, http.client.HTTPException) as err:     # no connection, a timeout, a dropped answer
        reason = getattr(err, "reason", None) or err
        hint = ("Your Python cannot check the API's certificate: see Troubleshooting in the guide."
                if "CERTIFICATE_VERIFY_FAILED" in str(reason) else "Check your internet connection, then run it again.")
        sys.exit(f"Could not reach the API at {BASE}: {reason}. {hint}")
    changed = 0
    for call in fetched:
        old = state["calls"].get(call["id"])
        call["transcript_file"] = write_transcript(folder, call)
        status = "new" if old is None else ("same" if old == call else "changed")
        changed += status != "same"
        state["calls"][call["id"]] = call
        state["complete"][call["id"]] = is_complete(call, state["turns"].get(call["id"]))
        when = (call["started_at"] or call["ended_at"])[:16].replace("T", " ")
        who, need = call["caller"]["name"] or "Unknown caller", call["need"] or "need not known"
        print(f"{status:<8} {when}  {call['outcome']:<10}  {who}: {need}")
    try:
        write_csv(folder, state["calls"].values())
    except PermissionError:
        sys.exit("Could not replace calls.csv. Close it in Excel, then run the script again.")
    state["cursor"] = cursor
    save_state(folder, state)
    print(f"{len(state['calls'])} calls in {os.path.join(folder, 'calls.csv')} ({changed} new or changed)")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit("Stopped. Nothing is lost: run it again to continue.")