#!/usr/bin/env bash
# =============================================================================
# JRA-VAN（JV-Link）から 2020年〜当年 までの生データを取得する【唯一の正規手順】
# =============================================================================
#
# JV-Link の option=1（蓄積）の from_datetime は「更新日がこの日以降」であり開催年ではない。
# option=1 だけでは 2020 年 RA が数十件止まり・オッズ差分が膨らむだけになる。
#
#   フェーズ1: --all-stored --from-datetime 20200101（蓄積系＋RACE 差分）
#   フェーズ2: 2020〜当年 各年について **セットアップ option=4**
#             → RACE from YYYY0101 to YYYY1231（開催日範囲で RA/SE 等を取得）
#             → 当年のみ weekly-odds（オッズ差分・option=1）
#
# 取得中・取得後とも TRUNCATE / jv_delete_* はこのスクリプトからは行わない。
# 完了後: step04 / jv_normalize_past_years.sh（JV_NORMALIZE_MAX_YEAR=当年）
#
# 例:
#   nohup ./scripts/jv_wait_tunnel_and_fetch_raw.sh >> logs/jv_wait_fetch.nohup.log 2>&1 &
#
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
LOCK="${ROOT}/logs/.jv_fetch_2020_to_now.lock"
exec 9>"$LOCK"
if ! flock -n 9; then
  echo "ERROR: 別の jv_fetch_2020_to_now が実行中です。二重起動しないでください。" >&2
  exit 4
fi
# shellcheck source=/dev/null
source "${ROOT}/scripts/jv_load_env.sh"
cd "$ROOT"
export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}"

LOG_DIR="${ROOT}/logs"
mkdir -p "$LOG_DIR"
STAMP="$(TZ=Asia/Tokyo date +%Y%m%d_%H%M%S)"
LOG="${LOG_DIR}/jv_fetch_2020_to_now.${STAMP}.log"

PY=(python3.11)
command -v "${PY[0]}" &>/dev/null || PY=(python3)

FROM_YEAR="${JV_BACKFILL_FROM_YEAR:-2020}"
TO_YEAR="${JV_BACKFILL_TO_YEAR:-2026}"
ALL_STORED_FROM="${JV_FROM_DATETIME:-20200101}"
SETUP_OPTION="${JV_SETUP_OPTION:-4}"
HOST="${JVLINK_SERVER_HOST:-127.0.0.1}"
PORT="${JVLINK_SERVER_PORT:-8765}"
EXTRA_FETCH=()
if [[ -n "${JV_FETCH_EXTRA:-}" ]]; then
  # shellcheck disable=SC2206
  EXTRA_FETCH=(${JV_FETCH_EXTRA})
fi

log() {
  echo "[$(TZ=Asia/Tokyo date '+%F %T')] $*" | tee -a "$LOG"
}

ra_count() {
  local y="$1"
  # JSON_EXTRACT 全件集計は 1.6M 行で 30〜60 分かかり取得が止まって見えるためタイムアウトする
  timeout "${JV_RA_COUNT_TIMEOUT_SEC:-45}" mysql keiba_ai -N -e \
    "SELECT COUNT(*) FROM jv_raw_records WHERE record_type='RA' AND payload_json LIKE '%\"meet_year\": \"${y}\"%';" \
    2>/dev/null || echo "?"
}

log "=== JRA-VAN fetch ${FROM_YEAR}..${TO_YEAR} all_stored_from=${ALL_STORED_FROM} log=${LOG} ==="

if [[ -f "${ROOT}/config/JV_FETCH_PAUSED" ]]; then
  log "ERROR: config/JV_FETCH_PAUSED があります。rm してから再実行。"
  exit 3
fi

if ! "${PY[@]}" scripts/jv_link_tcp_healthcheck.py >>"$LOG" 2>&1; then
  log "ERROR: JV-Link 未接続。Windows: step01_start_tunnel.bat + JVLinkServer を開いたままに。"
  exit 2
fi

if [[ "${JV_SKIP_ALL_STORED:-0}" == "1" ]]; then
  log "JV_SKIP_ALL_STORED=1: skip phase 1 (phase 2 のみ)"
else
  log "--- phase 1/2: --all-stored from ${ALL_STORED_FROM} ---"
  if ! "${PY[@]}" collectors/fetch_jv_raw.py \
    "${EXTRA_FETCH[@]}" --host "$HOST" --port "$PORT" \
    --from-datetime "$ALL_STORED_FROM" --all-stored \
    >>"$LOG" 2>&1; then
    log "WARN: phase 1 (--all-stored) failed — phase 2 (2020..${TO_YEAR} RACE) を続行"
  fi
  race1=$(mysql keiba_ai -N -e "SELECT COUNT(*) FROM jv_raw_records WHERE dataspec='RACE';" 2>/dev/null || echo "?")
  log "phase 1 done raw=$(mysql keiba_ai -N -e 'SELECT COUNT(*) FROM jv_raw_records;' 2>/dev/null || echo '?') RACE_dataspec=${race1}"
fi

log "--- phase 2/2: per-year RACE option=1 ${FROM_YEAR}..${TO_YEAR} ---"
for ((y = FROM_YEAR; y <= TO_YEAR; y++)); do
  race_from="${y}0101"

  log "year ${y}: RACE from ${race_from} (option=1 — 6/5 成功ログと同じ)"
  if ! "${PY[@]}" collectors/fetch_jv_raw.py \
    "${EXTRA_FETCH[@]}" --host "$HOST" --port "$PORT" \
    --from-datetime "$race_from" --dataspec RACE --option 1 \
    --busy-retries "${JV_BUSY_RETRIES:-30}" --busy-sleep "${JV_BUSY_SLEEP_SEC:-20}" \
    >>"$LOG" 2>&1; then
    log "WARN: RACE fetch failed year=${y}"
  fi
  sleep "${JV_BETWEEN_FETCH_SEC:-5}"

  if (( y == TO_YEAR )); then
    log "year ${y}: weekly-odds (current year only)"
    if ! "${PY[@]}" collectors/fetch_jv_raw.py \
      "${EXTRA_FETCH[@]}" --host "$HOST" --port "$PORT" \
      --from-datetime "$race_from" --weekly-odds \
      >>"$LOG" 2>&1; then
      log "WARN: weekly-odds failed year=${y}"
    fi
    sleep "${JV_BETWEEN_FETCH_SEC:-5}"
  fi

  race_total=$(mysql keiba_ai -N -e "SELECT COUNT(*) FROM jv_raw_records WHERE dataspec='RACE';" 2>/dev/null || echo '?')
  ra_y=$(ra_count "$y")
  log "year ${y} checkpoint: RA=${ra_y} RACE_total=${race_total}"
done

RAW=$(mysql keiba_ai -N -e "SELECT COUNT(*) FROM jv_raw_records;" 2>/dev/null || echo "?")
RACE_DS=$(mysql keiba_ai -N -e "SELECT COUNT(*) FROM jv_raw_records WHERE dataspec='RACE';" 2>/dev/null || echo "?")
log "fetch done raw=${RAW} RACE_dataspec=${RACE_DS}"

log "--- NORMALIZE (from-id=${JV_NORMALIZE_FROM_ID:-0}) ---"
"${PY[@]}" collectors/normalize_jv_raw.py \
  --from-id "${JV_NORMALIZE_FROM_ID:-0}" \
  --batch-size "${JV_NORMALIZE_BATCH:-5000}" \
  >>"$LOG" 2>&1

log "--- MATERIALIZE ---"
"${PY[@]}" collectors/materialize_jv_app_records.py >>"$LOG" 2>&1

RACES=$(mysql keiba_ai -N -e "SELECT COUNT(*) FROM races WHERE circuit='JRA';" 2>/dev/null || echo "?")
log "=== DONE races(JRA)=${RACES} raw=${RAW} ==="
log "確認: scripts/audit_data_coverage.py --from-year ${FROM_YEAR} --to-year ${TO_YEAR}"