import "server-only";

import { chromium } from "playwright";
import { LicitationProcedureType, LicitationSource, MunicipalOpenPortalType } from "@prisma/client";
import { licitationsConfig } from "@/lib/licitations/config";
import { mapCategory, mapProcedureType, parseAmountAndCurrency, parseDateMaybe, pickFirstText, sanitizeDocuments } from "@/lib/licitations/normalize";
import type { ConnectorResult, MunicipalityConnectorInput, NormalizedLicitationInput } from "@/lib/licitations/types";
import { delay, normalizeUrl, withRetries } from "@/lib/licitations/utils";

const MUNICIPAL_KEYWORDS = [
  "licitacion",
  "licitaciones",
  "convocatoria",
  "convocatorias",
  "adquisiciones",
  "obra publica",
  "fraccion xxviii",
  "fraccion xxix",
  "fallo",
  "bases",
  "anexo",
];

const OPEN_STATUS_TERMS = ["abierta", "vigente", "en curso", "recepcion", "publicada", "convocada", "activa"];
const SAN_PEDRO_TENDER_CODE_PATTERN = /\b[A-Z]{2,}(?:-[A-Z0-9]+){2,}-\d{1,4}\/\d{4}\b/i;
const ANTIBOT_PATTERN =
  /(just a moment|cloudflare|connection timed out|error code 52\d|enable javascript and cookies|attention required)/i;

type AspxPostbackTarget = {
  target: string;
  argument: string;
  label: string;
};

type MunicipalSnapshot = {
  pageUrl: string;
  pageTitle: string;
  frameCount: number;
  tablesCount: number;
  iframeSrcs: string[];
  apiRecords: Array<{
    url: string;
    contentType: string;
    payload: unknown;
  }>;
  links: Array<{ name: string; url: string; context: string }>;
  bodyText: string;
  hasAntiBotChallenge: boolean;
  postbackTargets: AspxPostbackTarget[];
  tableRows: Array<{
    tableIndex: number;
    rowIndex: number;
    values: Record<string, string>;
    links: Array<{ name: string; url: string }>;
    rowText: string;
  }>;
};

function pickRecordValue(record: Record<string, unknown>, patterns: RegExp[]) {
  for (const [key, value] of Object.entries(record)) {
    if (!patterns.some((pattern) => pattern.test(key))) {
      continue;
    }

    if (typeof value === "string" && value.trim()) {
      return value.trim();
    }

    if (typeof value === "number" || typeof value === "boolean") {
      return String(value);
    }
  }

  return null;
}

function extractDocumentUrls(value: unknown, found: Set<string>) {
  if (typeof value === "string") {
    const trimmed = value.trim();
    if (/\.(pdf|xls|xlsx|csv|doc|docx|zip)(?:$|\?)/i.test(trimmed) || /^https?:\/\//i.test(trimmed)) {
      found.add(trimmed);
    }
    return;
  }

  if (Array.isArray(value)) {
    for (const item of value) {
      extractDocumentUrls(item, found);
    }
    return;
  }

  if (value && typeof value === "object") {
    for (const item of Object.values(value as Record<string, unknown>)) {
      extractDocumentUrls(item, found);
    }
  }
}

function extractCandidateObjects(payload: unknown, out: Array<Record<string, unknown>>, depth = 0) {
  if (depth > 8 || payload == null) {
    return;
  }

  if (Array.isArray(payload)) {
    for (const item of payload) {
      extractCandidateObjects(item, out, depth + 1);
    }
    return;
  }

  if (typeof payload === "object") {
    const record = payload as Record<string, unknown>;
    const keys = Object.keys(record);
    const joinedKeys = keys.join(" ").toLowerCase();
    const looksLikeLicitation =
      /(licit|convocat|adjud|invitaci|procedim|contrat|compra|obra|servicio|adquis|monto|folio|expediente|fecha)/i.test(joinedKeys);

    if (looksLikeLicitation) {
      out.push(record);
    }

    for (const value of Object.values(record)) {
      extractCandidateObjects(value, out, depth + 1);
    }
  }
}

function buildFromApiRecord(
  snapshot: MunicipalSnapshot,
  apiRecord: { url: string; contentType: string; payload: unknown },
  options: { forcedProcedureType?: LicitationProcedureType; forceOpen?: boolean },
) {
  const records: Array<Record<string, unknown>> = [];
  extractCandidateObjects(apiRecord.payload, records);

  const items: NormalizedLicitationInput[] = [];

  for (const record of records) {
    const textBlob = Object.values(record)
      .map((value) => (typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? String(value) : ""))
      .join(" | ")
      .trim();

    const title = pickFirstText([
      pickRecordValue(record, [/titulo/i, /nombre/i, /objeto/i, /descrip/i, /concepto/i, /licitaci[oó]n/i]),
      null,
    ]);

    if (!title) {
      continue;
    }

    const amountText = pickFirstText([
      pickRecordValue(record, [/monto/i, /importe/i, /presupuesto/i, /total/i]),
      textBlob,
    ]);
    const amountData = parseAmountAndCurrency(amountText);

    const publishDate = parseDateMaybe(
      pickFirstText([pickRecordValue(record, [/publicaci[oó]n/i, /^fecha$/i, /fec_pub/i]), null]),
    );

    const openingDate = parseDateMaybe(
      pickFirstText([pickRecordValue(record, [/apertura/i, /inicio/i, /junta/i]), null]),
    );

    const closingDate = parseDateMaybe(
      pickFirstText([pickRecordValue(record, [/cierre/i, /l[ií]mite/i, /vencim/i, /recepci[oó]n/i]), null]),
    );

    const status = pickFirstText([pickRecordValue(record, [/estatus/i, /estado/i, /situaci[oó]n/i, /vigencia/i]), null]) || null;
    const tenderCode = pickFirstText([
      pickRecordValue(record, [/folio/i, /expediente/i, /n[uú]mero/i, /^id$/i, /clave/i, /codigo/i]),
      null,
    ]);

    const documentsSet = new Set<string>();
    extractDocumentUrls(record, documentsSet);
    const documents = [...documentsSet]
      .map((url) => normalizeUrl(url, snapshot.pageUrl))
      .filter((url): url is string => Boolean(url))
      .map((url) => ({ name: "Documento", url }));

    const hasSignal =
      includesKeyword(`${title} ${textBlob}`) ||
      documents.length > 0 ||
      amountData.amount !== null ||
      status !== null ||
      (options.forcedProcedureType && options.forcedProcedureType !== LicitationProcedureType.UNKNOWN);

    if (!hasSignal) {
      continue;
    }

    const procedureType =
      options.forcedProcedureType && options.forcedProcedureType !== LicitationProcedureType.UNKNOWN
        ? options.forcedProcedureType
        : mapProcedureType(`${title} ${textBlob}`);

    const openByDates = !closingDate || closingDate >= new Date();
    const isOpen = options.forceOpen === true ? true : looksOpen(status) || openByDates;

    items.push({
      sourceRecordId: pickFirstText([tenderCode, documents[0]?.url, `${apiRecord.url}#${title.slice(0, 120)}`]),
      tenderCode: tenderCode || null,
      procedureType,
      title,
      description: pickFirstText([pickRecordValue(record, [/descrip/i, /detalle/i, /observ/i]), textBlob]) || null,
      category: mapCategory(`${title} ${textBlob}`),
      isOpen,
      openingDate,
      closingDate,
      publishDate,
      amount: amountData.amount,
      currency: amountData.currency,
      status,
      documents: sanitizeDocuments(documents),
      rawSourceUrl: snapshot.pageUrl,
      rawPayload: {
        mode: "api-record",
        apiUrl: apiRecord.url,
        apiContentType: apiRecord.contentType,
        record,
      },
    } satisfies NormalizedLicitationInput);
  }

  return items;
}

function includesKeyword(value: string) {
  const normalized = value
    .toLowerCase()
    .normalize("NFD")
    .replace(/\p{Diacritic}/gu, "");

  return MUNICIPAL_KEYWORDS.some((keyword) => normalized.includes(keyword));
}

function looksOpen(value: string | null | undefined) {
  const normalized = (value ?? "").toLowerCase();
  return OPEN_STATUS_TERMS.some((term) => normalized.includes(term));
}

function pickRowValue(row: MunicipalSnapshot["tableRows"][number], patterns: RegExp[]) {
  for (const [key, value] of Object.entries(row.values)) {
    if (patterns.some((pattern) => pattern.test(key))) {
      const cleaned = value.trim();

      if (cleaned) {
        return cleaned;
      }
    }
  }

  return null;
}

function parseProcedureFromProParam(pro: string | null | undefined) {
  if (pro === "1") {
    return LicitationProcedureType.LICITACION_PUBLICA;
  }

  if (pro === "2") {
    return LicitationProcedureType.INVITACION_RESTRINGIDA;
  }

  if (pro === "3") {
    return LicitationProcedureType.ADJUDICACION_DIRECTA;
  }

  return LicitationProcedureType.UNKNOWN;
}

function buildStubFromLink(snapshot: MunicipalSnapshot, link: { name: string; url: string; context: string }, isOpen: boolean) {
  const url = normalizeUrl(link.url, snapshot.pageUrl);

  if (!url) {
    return null;
  }

  const title = pickFirstText([link.name, link.context, "Convocatoria municipal"]);

  return {
    sourceRecordId: url,
    procedureType: mapProcedureType(`${link.name} ${link.context}`),
    title,
    description: "Registro generado desde enlace documental municipal.",
    category: mapCategory(`${link.name} ${link.context}`),
    publishDate: parseDateMaybe(link.context),
    isOpen,
    documents: sanitizeDocuments([{ name: title, url }]),
    rawSourceUrl: snapshot.pageUrl,
    rawPayload: {
      mode: isOpen ? "open-document-feed" : "backup-document-feed",
      link,
    },
  } satisfies NormalizedLicitationInput;
}

function buildFromTableRow(
  snapshot: MunicipalSnapshot,
  row: MunicipalSnapshot["tableRows"][number],
  options: {
    forcedProcedureType?: LicitationProcedureType;
    forceOpen?: boolean;
  },
) {
  const title = pickFirstText([
    pickRowValue(row, [/objeto/i, /descripcion/i, /descripci[oó]n/i, /concepto/i, /licitaci[oó]n/i, /nombre/i]),
    row.rowText,
  ]);

  if (!title) {
    return null;
  }

  const amountText = pickFirstText([pickRowValue(row, [/monto/i, /importe/i, /presupuesto/i]), row.rowText]);
  const amountData = parseAmountAndCurrency(amountText);

  const publishDate = parseDateMaybe(
    pickFirstText([
      pickRowValue(row, [/fecha de publicaci[oó]n/i, /publicaci[oó]n/i, /^fecha$/i]),
      pickRowValue(row, [/convocatoria/i]),
    ]),
  );

  const openingDate = parseDateMaybe(
    pickFirstText([pickRowValue(row, [/apertura/i, /junta/i, /inicio/i]), pickRowValue(row, [/fecha/i])]),
  );

  const closingDate = parseDateMaybe(
    pickFirstText([pickRowValue(row, [/cierre/i, /l[ií]mite/i, /recepci[oó]n/i, /propuesta/i]), null]),
  );

  const status = pickFirstText([pickRowValue(row, [/estatus/i, /estado/i, /situaci[oó]n/i]), null]) || null;

  const procedureType =
    options.forcedProcedureType && options.forcedProcedureType !== LicitationProcedureType.UNKNOWN
      ? options.forcedProcedureType
      : mapProcedureType(pickFirstText([pickRowValue(row, [/procedimiento/i, /tipo/i]), row.rowText]));

  const documents = row.links
    .map((link) => {
      const url = normalizeUrl(link.url, snapshot.pageUrl);
      return url ? { name: link.name || "Documento", url } : null;
    })
    .filter((item): item is { name: string; url: string } => Boolean(item));

  const tenderCode = pickFirstText([
    pickRowValue(row, [/folio/i, /expediente/i, /n[uú]mero/i, /^no\.?$/i, /clave/i]),
    null,
  ]);

  const openByDates = !closingDate || closingDate >= new Date();
  const isOpen = options.forceOpen === true ? true : looksOpen(status) || openByDates;

  return {
    sourceRecordId: pickFirstText([
      tenderCode,
      documents[0]?.url,
      `${snapshot.pageUrl}#table-${row.tableIndex}-row-${row.rowIndex}`,
    ]),
    tenderCode: tenderCode || null,
    procedureType,
    title,
    description: pickFirstText([pickRowValue(row, [/descripci[oó]n/i, /detalle/i, /observaciones/i]), null]) || null,
    category: mapCategory(pickFirstText([pickRowValue(row, [/categor[ií]a/i, /rubro/i, /giro/i]), row.rowText])),
    isOpen,
    openingDate,
    closingDate,
    publishDate,
    amount: amountData.amount,
    currency: amountData.currency,
    status,
    documents: sanitizeDocuments(documents),
    rawSourceUrl: snapshot.pageUrl,
    rawPayload: {
      mode: "table-row",
      row,
    },
  } satisfies NormalizedLicitationInput;
}

function safePreview(value: unknown) {
  try {
    return JSON.stringify(value).slice(0, 380);
  } catch {
    return String(value).slice(0, 380);
  }
}

function slugifyRecordIdPart(value: string) {
  return value
    .toLowerCase()
    .normalize("NFD")
    .replace(/\p{Diacritic}/gu, "")
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-+|-+$/g, "")
    .slice(0, 80);
}

function getPostbackPriority(target: AspxPostbackTarget) {
  const text = `${target.target} ${target.argument} ${target.label}`.toLowerCase();
  let score = 0;

  if (/buscar|consultar|filtrar|aplicar|actualizar|mostrar/.test(text)) {
    score += 100;
  }

  if (/page\$|next|siguiente|pagin/.test(text)) {
    score += 60;
  }

  if (/grid|gv|repeater|list|datos|result/.test(text)) {
    score += 40;
  }

  if (text.length <= 6) {
    score -= 20;
  }

  return score;
}

function mergeMunicipalSnapshots(base: MunicipalSnapshot, incoming: MunicipalSnapshot): MunicipalSnapshot {
  const linkMap = new Map<string, MunicipalSnapshot["links"][number]>();
  for (const link of [...base.links, ...incoming.links]) {
    linkMap.set(`${link.url}|${link.name}|${link.context}`, link);
  }

  const rowMap = new Map<string, MunicipalSnapshot["tableRows"][number]>();
  for (const row of [...base.tableRows, ...incoming.tableRows]) {
    const rowLinksKey = row.links
      .map((link) => `${link.name}|${link.url}`)
      .sort()
      .join("|");
    rowMap.set(`${row.rowText}|${rowLinksKey}`, row);
  }

  const apiMap = new Map<string, MunicipalSnapshot["apiRecords"][number]>();
  for (const apiRecord of [...base.apiRecords, ...incoming.apiRecords]) {
    apiMap.set(`${apiRecord.url}|${apiRecord.contentType}|${safePreview(apiRecord.payload)}`, apiRecord);
  }

  const postbackMap = new Map<string, AspxPostbackTarget>();
  for (const postback of [...base.postbackTargets, ...incoming.postbackTargets]) {
    postbackMap.set(`${postback.target}|${postback.argument}`, postback);
  }

  const mergedBodyText = incoming.bodyText.length > base.bodyText.length ? incoming.bodyText : base.bodyText;
  const mergedPageTitle =
    !incoming.hasAntiBotChallenge && incoming.pageTitle ? incoming.pageTitle : base.pageTitle || incoming.pageTitle;

  return {
    pageUrl: incoming.pageUrl || base.pageUrl,
    pageTitle: mergedPageTitle,
    frameCount: Math.max(base.frameCount, incoming.frameCount),
    tablesCount: Math.max(base.tablesCount, incoming.tablesCount),
    iframeSrcs: [...new Set([...base.iframeSrcs, ...incoming.iframeSrcs])],
    apiRecords: [...apiMap.values()],
    links: [...linkMap.values()],
    bodyText: mergedBodyText,
    hasAntiBotChallenge: base.hasAntiBotChallenge || incoming.hasAntiBotChallenge,
    postbackTargets: [...postbackMap.values()],
    tableRows: [...rowMap.values()],
  };
}

function extractSanPedroTextBlocks(sourceText: string) {
  const normalized = sourceText
    .replace(/\u00a0/g, " ")
    .replace(/\r/g, "")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();

  if (!normalized) {
    return [];
  }

  const directBlocks = [...normalized.matchAll(/Concurso\s*-[\s\S]{80,2200}?(?=(?:\n\s*Concurso\s*-|$))/gi)]
    .map((match) => (match[0] ?? "").trim())
    .filter((value) => value.length >= 60);

  const fallbackBlocks =
    directBlocks.length > 0
      ? []
      : [...normalized.matchAll(/No\.?\s*Procedimiento[\s\S]{80,1800}?(?=(?:\n\s*No\.?\s*Procedimiento|$))/gi)]
          .map((match) => (match[0] ?? "").trim())
          .filter((value) => value.length >= 60);

  const deduped = new Map<string, string>();
  for (const block of [...directBlocks, ...fallbackBlocks]) {
    const key = block
      .toLowerCase()
      .replace(/\s+/g, " ")
      .slice(0, 260);
    deduped.set(key, block);
  }

  return [...deduped.values()];
}

function buildFromSanPedroTextBlock(
  snapshot: MunicipalSnapshot,
  block: string,
  blockIndex: number,
  options: {
    forcedProcedureType?: LicitationProcedureType;
    forceOpen?: boolean;
  },
) {
  const condensed = block.replace(/\s+/g, " ").trim();

  if (condensed.length < 40 || !/(concurso|procedimiento|licitaci[oó]n)/i.test(condensed)) {
    return null;
  }

  const tenderCode = pickFirstText([condensed.match(SAN_PEDRO_TENDER_CODE_PATTERN)?.[0], null]) || null;
  const title = pickFirstText([
    condensed.match(/Concurso\s*-\s*(.+?)(?=No\.?\s*Procedimiento|TIPO DE PROCEDIMIENTO|EN PROCESO|Descripci[oó]n|$)/i)?.[1],
    condensed.match(/Descripci[oó]n\s*:?\s*(.+?)(?=Fecha|No\.?\s*Procedimiento|TIPO DE PROCEDIMIENTO|$)/i)?.[1],
    tenderCode ? `Concurso ${tenderCode}` : null,
  ]);

  if (!title) {
    return null;
  }

  const description =
    pickFirstText([
      condensed.match(
        /Descripci[oó]n\s*:?\s*([\s\S]{1,700}?)(?=Fecha de|Fecha de Actualizaci[oó]n|No\.?\s*Procedimiento|TIPO DE PROCEDIMIENTO|$)/i,
      )?.[1],
      null,
    ]) || null;

  const procedureText = pickFirstText([
    condensed.match(
      /TIPO DE PROCEDIMIENTO\s*:?\s*([\s\S]{1,220}?)(?=EN PROCESO|ABIERTA|VIGENTE|Descripci[oó]n|Fecha|No\.?\s*Procedimiento|$)/i,
    )?.[1],
    condensed,
  ]);

  const status = pickFirstText([
    condensed.match(/\b(EN PROCESO|VIGENTE|ABIERTA?|PUBLICADA?|CERRADA?|CANCELADA?|DESIERTA|CONCLUIDA)\b/i)?.[1],
    null,
  ]);

  const publishDate = parseDateMaybe(
    pickFirstText([
      condensed.match(/Fecha de Actualizaci[oó]n\s*:?\s*([0-3]?\d[\/.-][01]?\d[\/.-]\d{2,4})/i)?.[1],
      condensed.match(/Fecha de Publicaci[oó]n\s*:?\s*([0-3]?\d[\/.-][01]?\d[\/.-]\d{2,4})/i)?.[1],
      null,
    ]),
  );

  const openingDate = parseDateMaybe(
    pickFirstText([
      condensed.match(/(?:Fecha de Apertura|Inicio)\s*:?\s*([0-3]?\d[\/.-][01]?\d[\/.-]\d{2,4})/i)?.[1],
      null,
    ]),
  );

  const closingDate = parseDateMaybe(
    pickFirstText([
      condensed.match(
        /(?:Fecha L[ií]mite|Fecha de Cierre|Cierre|Recepci[oó]n de Propuestas)\s*:?\s*([0-3]?\d[\/.-][01]?\d[\/.-]\d{2,4})/i,
      )?.[1],
      null,
    ]),
  );

  const amountData = parseAmountAndCurrency(condensed);
  const lowerTenderCode = tenderCode?.toLowerCase() ?? "";
  const documentLinks = snapshot.links.filter((link) => /\.(pdf|xls|xlsx|csv|doc|docx|zip)(?:$|\?)/i.test(link.url));

  const matchedDocuments = documentLinks
    .filter((link) => {
      if (!lowerTenderCode) {
        return true;
      }

      const blob = `${link.name} ${link.context}`.toLowerCase();
      return blob.includes(lowerTenderCode);
    })
    .slice(0, 12)
    .map((link) => {
      const normalizedUrl = normalizeUrl(link.url, snapshot.pageUrl);
      return normalizedUrl ? { name: link.name || "Documento", url: normalizedUrl } : null;
    })
    .filter((item): item is { name: string; url: string } => Boolean(item));

  const procedureType =
    options.forcedProcedureType && options.forcedProcedureType !== LicitationProcedureType.UNKNOWN
      ? options.forcedProcedureType
      : mapProcedureType(procedureText);

  const openByDates = !closingDate || closingDate >= new Date();
  const inferredOpenByStatus = looksOpen(status) || /\ben proceso\b/i.test(condensed);
  const isOpen = options.forceOpen === true ? true : inferredOpenByStatus || openByDates;
  const recordSlug = slugifyRecordIdPart(tenderCode || title || `block-${blockIndex + 1}`);

  return {
    sourceRecordId: pickFirstText([tenderCode, matchedDocuments[0]?.url, `${snapshot.pageUrl}#text-${recordSlug}`]),
    tenderCode: tenderCode || null,
    procedureType,
    title,
    description,
    category: mapCategory(`${title} ${description ?? ""}`),
    isOpen,
    openingDate,
    closingDate,
    publishDate,
    amount: amountData.amount,
    currency: amountData.currency,
    status: status || null,
    documents: sanitizeDocuments(matchedDocuments),
    rawSourceUrl: snapshot.pageUrl,
    rawPayload: {
      mode: "san-pedro-text-block",
      block,
      blockIndex,
    },
  } satisfies NormalizedLicitationInput;
}

function buildSanPedroTextItems(
  snapshot: MunicipalSnapshot,
  options: {
    forcedProcedureType?: LicitationProcedureType;
    forceOpen?: boolean;
  },
) {
  const bodyBlocks = extractSanPedroTextBlocks(snapshot.bodyText);
  const contextBlocks =
    bodyBlocks.length > 0
      ? []
      : extractSanPedroTextBlocks(
          snapshot.links
            .map((link) => link.context)
            .filter((value) => value.length >= 40)
            .join("\n"),
        );

  const blocks = bodyBlocks.length > 0 ? bodyBlocks : contextBlocks;
  const items: NormalizedLicitationInput[] = [];

  for (const [index, block] of blocks.entries()) {
    const parsed = buildFromSanPedroTextBlock(snapshot, block, index, options);

    if (parsed) {
      items.push(parsed);
    }
  }

  return items;
}

async function scrapePortalPage(
  url: string,
  options?: {
    enableAspxPostbacks?: boolean;
  },
) {
  const browser = await chromium.launch({ headless: true });

  try {
    const context = await browser.newContext({
      locale: "es-MX",
      timezoneId: "America/Monterrey",
      userAgent:
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    });
    const page = await context.newPage();
    const apiRecords: Array<{ url: string; contentType: string; payload: unknown }> = [];
    const apiTasks: Promise<void>[] = [];
    const origin = new URL(url).origin;

    await page.addInitScript(() => {
      Object.defineProperty(navigator, "webdriver", {
        get: () => undefined,
      });
    });

    page.on("response", (response) => {
      const task = (async () => {
        try {
          const responseUrl = response.url();
          if (!responseUrl.startsWith(origin)) {
            return;
          }

          const headers = response.headers();
          const contentType = headers["content-type"] || "";
          const looksInteresting =
            /json|javascript|xml|plain/i.test(contentType) ||
            /(licit|convocat|adjud|invitaci|contrat|compra|obra|servicio|adquis|api|webservice|asmx)/i.test(responseUrl);

          if (!looksInteresting) {
            return;
          }

          const body = await response.text();
          if (!body || body.length < 2) {
            return;
          }

          let payload: unknown = null;
          try {
            payload = JSON.parse(body);
          } catch {
            const jsonLike = body.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
            if (jsonLike) {
              try {
                payload = JSON.parse(jsonLike[1]);
              } catch {
                payload = null;
              }
            }
          }

          if (payload !== null) {
            apiRecords.push({
              url: responseUrl,
              contentType,
              payload,
            });
          }
        } catch {
          // Ignore response parsing failures.
        }
      })();

      apiTasks.push(task);
    });

    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45_000 });
    await page.waitForLoadState("networkidle", { timeout: 12_000 }).catch(() => undefined);
    await page.waitForTimeout(2200);
    await Promise.allSettled(apiTasks);

    const collectSnapshot = async (target: { evaluate: <T>(fn: () => T) => Promise<T> }) => {
      return target.evaluate(() => {
        const normalize = (value: string | null | undefined) => (value ?? "").replace(/\s+/g, " ").trim();
        const cleanBodyText = (value: string | null | undefined) =>
          (value ?? "")
            .replace(/\u00a0/g, " ")
            .replace(/\r/g, "")
            .replace(/[ \t]+\n/g, "\n")
            .replace(/\n{3,}/g, "\n\n")
            .trim();

        const extractPostback = (raw: string) => {
          const match = raw.match(/__doPostBack\((['"])(.*?)\1\s*,\s*(['"])(.*?)\3\)/i);
          if (!match) {
            return null;
          }

          return {
            target: (match[2] ?? "").trim(),
            argument: (match[4] ?? "").trim(),
          };
        };

        const links = Array.from(document.querySelectorAll("a[href]")).map((anchor) => {
          const parent = anchor.closest("li, p, div, tr, article");
          return {
            name: normalize(anchor.textContent) || normalize(anchor.getAttribute("title")) || "Documento",
            url: String(anchor.getAttribute("href") || ""),
            context: normalize(parent?.textContent).slice(0, 1200),
          };
        });

        const tables = Array.from(document.querySelectorAll("table"));
        const tableRows = tables.flatMap((table, tableIndex) => {
          const headers = Array.from(table.querySelectorAll("thead th, tr th")).map(
            (header, index) => normalize(header.textContent) || `col_${index + 1}`,
          );

          return Array.from(table.querySelectorAll("tbody tr, tr"))
            .filter((row) => row.querySelectorAll("td").length > 0)
            .map((row, rowIndex) => {
              const cells = Array.from(row.querySelectorAll("td")).map((cell) => normalize(cell.textContent));
              const values: Record<string, string> = {};

              cells.forEach((cell, index) => {
                const key = headers[index] || `col_${index + 1}`;
                values[key] = cell;
              });

              const rowLinks = Array.from(row.querySelectorAll("a[href]"))
                .map((link) => ({
                  name: normalize(link.textContent) || "Documento",
                  url: String(link.getAttribute("href") || ""),
                }))
                .filter((item) => item.url.length > 0);

              return {
                tableIndex,
                rowIndex,
                values,
                links: rowLinks,
                rowText: cells.join(" | "),
              };
            });
        });

        const postbackMap = new Map<string, { target: string; argument: string; label: string }>();
        for (const element of Array.from(document.querySelectorAll("a[href], [onclick]"))) {
          const href = String(element.getAttribute("href") || "");
          const onclick = String(element.getAttribute("onclick") || "");
          const payload = extractPostback(href) || extractPostback(onclick);

          if (!payload?.target) {
            continue;
          }

          const label =
            normalize(element.textContent) ||
            normalize(element.getAttribute("title")) ||
            normalize(element.getAttribute("name")) ||
            normalize(element.getAttribute("id")) ||
            "postback";
          postbackMap.set(`${payload.target}|${payload.argument}`, {
            target: payload.target,
            argument: payload.argument,
            label,
          });
        }

        const pageTitle = document.title || "Municipality procurement";
        const bodyText = cleanBodyText(document.body?.innerText);
        const antiBotBlob = `${pageTitle} ${bodyText.slice(0, 2400)}`;
        const hasAntiBotChallenge =
          /(just a moment|cloudflare|connection timed out|error code 52\d|enable javascript and cookies|attention required)/i.test(
            antiBotBlob,
          );

        return {
          pageUrl: window.location.href,
          pageTitle,
          iframeSrcs: Array.from(document.querySelectorAll("iframe[src]"))
            .map((iframe) => String(iframe.getAttribute("src") || ""))
            .filter((value) => value.length > 0),
          links,
          tableRows,
          tablesCount: tables.length,
          bodyText,
          hasAntiBotChallenge,
          postbackTargets: [...postbackMap.values()],
        };
      });
    };

    const collectMergedSnapshot = async () => {
      const mainSnapshot = (await collectSnapshot(page)) as Omit<MunicipalSnapshot, "frameCount" | "apiRecords">;
      const childFrames = page.frames().filter((frame) => frame !== page.mainFrame());

      const mergedLinks = [...mainSnapshot.links];
      const mergedRows = [...mainSnapshot.tableRows];
      const mergedIframes = [...mainSnapshot.iframeSrcs];
      let tableOffset = mainSnapshot.tablesCount;
      let frameCount = 1;

      for (const frame of childFrames) {
        try {
          const frameSnapshot = (await collectSnapshot(frame)) as Omit<MunicipalSnapshot, "frameCount" | "apiRecords">;
          frameCount += 1;

          mergedLinks.push(
            ...frameSnapshot.links.map((link) => ({
              ...link,
              context: `${frameSnapshot.pageTitle} ${link.context}`.trim().slice(0, 1200),
            })),
          );

          mergedRows.push(
            ...frameSnapshot.tableRows.map((row) => ({
              ...row,
              tableIndex: row.tableIndex + tableOffset,
              rowText: `${frameSnapshot.pageTitle} | ${row.rowText}`.trim(),
            })),
          );

          mergedIframes.push(...frameSnapshot.iframeSrcs);
          tableOffset += frameSnapshot.tablesCount;
        } catch {
          // Ignore frame-specific parsing errors and continue with available content.
        }
      }

      return {
        pageUrl: mainSnapshot.pageUrl,
        pageTitle: mainSnapshot.pageTitle,
        links: mergedLinks,
        bodyText: mainSnapshot.bodyText,
        hasAntiBotChallenge: mainSnapshot.hasAntiBotChallenge,
        postbackTargets: mainSnapshot.postbackTargets,
        tableRows: mergedRows,
        tablesCount: tableOffset,
        frameCount,
        iframeSrcs: [...new Set(mergedIframes)],
        apiRecords: [...apiRecords],
      } satisfies MunicipalSnapshot;
    };

    const runPostback = async (postbackTarget: AspxPostbackTarget) => {
      const didTrigger = await page.evaluate(({ target, argument }) => {
        const maybeWindow = window as unknown as {
          __doPostBack?: (eventTarget: string, eventArgument: string) => void;
        };

        if (typeof maybeWindow.__doPostBack === "function") {
          maybeWindow.__doPostBack(target, argument);
          return true;
        }

        const form = document.querySelector("form");
        if (!(form instanceof HTMLFormElement)) {
          return false;
        }

        const ensureHidden = (name: string, value: string) => {
          let input = form.querySelector(`input[name="${name}"]`);

          if (!(input instanceof HTMLInputElement)) {
            input = document.createElement("input");
            input.setAttribute("type", "hidden");
            input.setAttribute("name", name);
            form.appendChild(input);
          }

          input.setAttribute("value", value);
        };

        ensureHidden("__EVENTTARGET", target);
        ensureHidden("__EVENTARGUMENT", argument);
        form.submit();
        return true;
      }, postbackTarget);

      if (!didTrigger) {
        return false;
      }

      await page.waitForLoadState("domcontentloaded", { timeout: 12_000 }).catch(() => undefined);
      await page.waitForLoadState("networkidle", { timeout: 12_000 }).catch(() => undefined);
      await page.waitForTimeout(1400);
      await Promise.allSettled(apiTasks);
      return true;
    };

    let snapshot = await collectMergedSnapshot();

    if (
      options?.enableAspxPostbacks &&
      !snapshot.hasAntiBotChallenge &&
      snapshot.tableRows.length === 0 &&
      snapshot.apiRecords.length === 0 &&
      snapshot.postbackTargets.length > 0
    ) {
      const candidates = [...snapshot.postbackTargets]
        .sort((left, right) => getPostbackPriority(right) - getPostbackPriority(left))
        .slice(0, 8);
      const seen = new Set<string>();

      for (const candidate of candidates) {
        const key = `${candidate.target}|${candidate.argument}`;

        if (seen.has(key)) {
          continue;
        }

        seen.add(key);

        const didTrigger = await runPostback(candidate);
        if (!didTrigger) {
          continue;
        }

        const postbackSnapshot = await collectMergedSnapshot();
        snapshot = mergeMunicipalSnapshots(snapshot, postbackSnapshot);
      }
    }

    await context.close();
    return snapshot;
  } finally {
    await browser.close();
  }
}

function buildSanPedroUrls(openPortalUrl: string, targetYear?: number) {
  const parsed = new URL(openPortalUrl);
  const yearFromUrl = Number.parseInt(parsed.searchParams.get("Year") ?? "", 10);
  const parsedYear = Number.isFinite(yearFromUrl) ? yearFromUrl : new Date().getUTCFullYear();
  const year = targetYear ?? parsedYear;
  const t = parsed.searchParams.get("T") ?? "1";
  const procedures = ["1", "2", "3"];
  const urls: string[] = [];

  for (const pro of procedures) {
    const url = new URL(parsed.toString());
    url.searchParams.set("Year", String(year));
    url.searchParams.set("T", t);
    url.searchParams.set("Pro", pro);
    urls.push(url.toString());
  }

  return urls;
}

function dedupeItems(items: NormalizedLicitationInput[]) {
  const map = new Map<string, NormalizedLicitationInput>();

  for (const item of items) {
    const key = item.sourceRecordId || `${item.title}|${item.rawSourceUrl || ""}`;
    map.set(key, item);
  }

  return [...map.values()];
}

function hasAntiBotSignal(snapshot: MunicipalSnapshot) {
  if (snapshot.hasAntiBotChallenge) {
    return true;
  }

  const probe = `${snapshot.pageTitle} ${snapshot.bodyText.slice(0, 2400)}`;
  return ANTIBOT_PATTERN.test(probe);
}

async function scrapeSanPedroOpenLicitations(municipality: MunicipalityConnectorInput, targetYear?: number) {
  if (!municipality.openPortalUrl) {
    throw new Error("Municipio sin openPortalUrl configurado.");
  }

  const candidateUrls = buildSanPedroUrls(municipality.openPortalUrl, targetYear);
  const items: NormalizedLicitationInput[] = [];
  const warnings: string[] = [];
  const candidateDebug: string[] = [];
  const antiBotCandidates = new Set<string>();

  for (const candidateUrl of candidateUrls) {
    const parsedUrl = new URL(candidateUrl);
    const forcedProcedureType = parseProcedureFromProParam(parsedUrl.searchParams.get("Pro"));
    const forceOpen = parsedUrl.searchParams.get("T") === "1";

    try {
      const snapshot = await scrapePortalPage(candidateUrl, { enableAspxPostbacks: true });
      const hasAntiBot = hasAntiBotSignal(snapshot);

      if (hasAntiBot) {
        antiBotCandidates.add(candidateUrl);
      }

      const tableItems = snapshot.tableRows
        .map((row) => buildFromTableRow(snapshot, row, { forcedProcedureType, forceOpen }))
        .filter(Boolean) as NormalizedLicitationInput[];
      const apiItems = snapshot.apiRecords.flatMap((record) =>
        buildFromApiRecord(snapshot, record, {
          forcedProcedureType,
          forceOpen,
        }),
      );
      const textItems = buildSanPedroTextItems(snapshot, { forcedProcedureType, forceOpen });

      const iframeItems: NormalizedLicitationInput[] = [];
      const iframeUrls = [...new Set(snapshot.iframeSrcs.map((src) => normalizeUrl(src, snapshot.pageUrl)).filter((value): value is string => Boolean(value)))];

      if (tableItems.length === 0 && apiItems.length === 0 && textItems.length === 0 && iframeUrls.length > 0) {
        for (const iframeUrl of iframeUrls.slice(0, 4)) {
          if (iframeUrl === candidateUrl) {
            continue;
          }

          try {
            const iframeSnapshot = await scrapePortalPage(iframeUrl, { enableAspxPostbacks: true });
            const iframeTableItems = iframeSnapshot.tableRows
              .map((row) => buildFromTableRow(iframeSnapshot, row, { forcedProcedureType, forceOpen }))
              .filter(Boolean) as NormalizedLicitationInput[];
            const iframeApiItems = iframeSnapshot.apiRecords.flatMap((record) =>
              buildFromApiRecord(iframeSnapshot, record, {
                forcedProcedureType,
                forceOpen,
              }),
            );
            const iframeTextItems = buildSanPedroTextItems(iframeSnapshot, { forcedProcedureType, forceOpen });
            const iframeStructured = [...iframeTableItems, ...iframeApiItems, ...iframeTextItems];

            if (iframeStructured.length > 0) {
              iframeItems.push(...iframeStructured);
            } else {
              const iframeLinkItems = iframeSnapshot.links
                .filter((link) => includesKeyword(`${link.name} ${link.context}`) || /\.(pdf|xls|xlsx|zip)(?:$|\?)/i.test(link.url))
                .map((link) => buildStubFromLink(iframeSnapshot, link, true))
                .filter(Boolean) as NormalizedLicitationInput[];

              iframeItems.push(...iframeLinkItems);
            }
          } catch {
            // Ignore iframe URL parsing errors and continue with remaining candidates.
          }
        }
      }

      let parsedCount = tableItems.length + apiItems.length + textItems.length + iframeItems.length;

      if (tableItems.length > 0 || apiItems.length > 0 || textItems.length > 0 || iframeItems.length > 0) {
        items.push(...tableItems);
        items.push(...apiItems);
        items.push(...textItems);
        items.push(...iframeItems);
      } else {
        const linkItems = snapshot.links
          .filter((link) => includesKeyword(`${link.name} ${link.context}`) || /\.(pdf|xls|xlsx|zip)(?:$|\?)/i.test(link.url))
          .map((link) => buildStubFromLink(snapshot, link, true))
          .filter(Boolean) as NormalizedLicitationInput[];

        parsedCount += linkItems.length;
        items.push(...linkItems);
      }

      candidateDebug.push(
        `${candidateUrl} [frames=${snapshot.frameCount}, tables=${snapshot.tablesCount}, rows=${snapshot.tableRows.length}, links=${snapshot.links.length}, api=${snapshot.apiRecords.length}, text=${textItems.length}, postbacks=${snapshot.postbackTargets.length}, iframes=${snapshot.iframeSrcs.length}, antiBot=${hasAntiBot ? 1 : 0}, parsed=${parsedCount}]`,
      );
    } catch (error) {
      warnings.push(`No se pudo leer ${candidateUrl}: ${error instanceof Error ? error.message : "error"}`);
    }

    await delay(licitationsConfig.requestDelayMs);
  }

  if (!items.length && antiBotCandidates.size > 0) {
    warnings.push(
      `El portal de San Pedro respondió con una página de protección/anti-bot (Cloudflare o equivalente) y no se pudo obtener contenido de licitaciones: ${[...antiBotCandidates].join(", ")}`,
    );
  }

  if (!items.length && candidateDebug.length > 0) {
    warnings.push(`No se detectaron registros en el portal de San Pedro. Debug: ${candidateDebug.join(" || ")}`);
  }

  return {
    source: LicitationSource.MUNICIPAL_OPEN_PORTAL,
    items: dedupeItems(items),
    rawSourceUrl: municipality.openPortalUrl,
    warnings,
  } satisfies ConnectorResult;
}

export async function fetchMunicipalOpenLicitations(
  municipality: MunicipalityConnectorInput,
  options?: {
    targetYear?: number;
  },
): Promise<ConnectorResult> {
  if (!municipality.openPortalUrl) {
    return {
      source: LicitationSource.MUNICIPAL_OPEN_PORTAL,
      items: [],
      warnings: ["Municipio sin openPortalUrl configurado."],
    };
  }

  if (municipality.openPortalType === MunicipalOpenPortalType.SAN_PEDRO_ASPX) {
    return withRetries(() => scrapeSanPedroOpenLicitations(municipality, options?.targetYear), {
      retries: licitationsConfig.maxRetries,
      initialDelayMs: 1200,
    });
  }

  const snapshot = await withRetries(() => scrapePortalPage(municipality.openPortalUrl as string), {
    retries: licitationsConfig.maxRetries,
    initialDelayMs: 900,
  });

  await delay(licitationsConfig.requestDelayMs);

  const tableItems = snapshot.tableRows
    .map((row) => buildFromTableRow(snapshot, row, {}))
    .filter(Boolean) as NormalizedLicitationInput[];
  const apiItems = snapshot.apiRecords.flatMap((record) => buildFromApiRecord(snapshot, record, {}));

  const documentFeedItems = snapshot.links
    .filter((link) => includesKeyword(`${link.name} ${link.context}`) || /\.(pdf|xls|xlsx|csv|doc|docx|zip)(?:$|\?)/i.test(link.url))
    .map((link) => buildStubFromLink(snapshot, link, true))
    .filter(Boolean) as NormalizedLicitationInput[];

  const parsedFromStructured = [...tableItems, ...apiItems];
  const items = parsedFromStructured.length ? parsedFromStructured : documentFeedItems;
  const antiBotSignal = hasAntiBotSignal(snapshot);
  const warnings = items.length
    ? []
    : [
        `No se detectaron licitaciones abiertas en el portal municipal. Debug: frames=${snapshot.frameCount}, tables=${snapshot.tablesCount}, rows=${snapshot.tableRows.length}, links=${snapshot.links.length}, postbacks=${snapshot.postbackTargets.length}, antiBot=${antiBotSignal ? 1 : 0}`,
      ];

  return {
    source: LicitationSource.MUNICIPAL_OPEN_PORTAL,
    items: dedupeItems(items),
    rawSourceUrl: snapshot.pageUrl,
    warnings,
  };
}

export async function fetchMunicipalBackupLicitations(municipality: MunicipalityConnectorInput): Promise<ConnectorResult> {
  const warnings: string[] = [];

  if (!municipality.backupUrl) {
    return {
      source: LicitationSource.MUNICIPAL_BACKUP,
      items: [],
      warnings: ["Municipio sin URL de respaldo configurada."],
    };
  }

  const snapshot = await withRetries(() => scrapePortalPage(municipality.backupUrl as string), {
    retries: licitationsConfig.maxRetries,
    initialDelayMs: 900,
  });

  await delay(licitationsConfig.requestDelayMs);

  const tableItems: NormalizedLicitationInput[] = [];
  for (const row of snapshot.tableRows) {
    const item = buildFromTableRow(snapshot, row, { forceOpen: false });
    if (item) {
      tableItems.push({ ...item, isOpen: false });
    }
  }

  const documentFeedItems = snapshot.links
    .filter((link) => {
      const text = `${link.name} ${link.context}`;
      const isKeywordHit = includesKeyword(text);
      const isDocument = /\.(pdf|xls|xlsx|csv|doc|docx|zip)(?:$|\?)/i.test(link.url);
      return isKeywordHit || isDocument;
    })
    .map((link) => buildStubFromLink(snapshot, link, false))
    .filter(Boolean) as NormalizedLicitationInput[];

  const items = tableItems.length ? tableItems : documentFeedItems;

  if (!items.length) {
    warnings.push("No se encontraron tablas o documentos relevantes en el portal municipal de respaldo.");
  }

  return {
    source: LicitationSource.MUNICIPAL_BACKUP,
    items: dedupeItems(items),
    rawSourceUrl: snapshot.pageUrl,
    warnings,
  };
}
