changes

scripts/backfill-licitaya-history.mjs

@@ -0,0 +1,578 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { PrismaClient, LicitationCategory, LicitationProcedureType, LicitationSource, MunicipalOpenPortalType } from "@prisma/client";
+
+const DEFAULT_ITEMS = 50;
+const DEFAULT_MAX_PAGES = 5;
+const DEFAULT_DAYS = 30;
+const DEFAULT_DELAY_MS = 400;
+const LICITAYA_MUNICIPALITY_CODE = "LICITAYA";
+
+const LICITAYA_STATE_MAP = {
+  AGS: { stateCode: "1", stateName: "Aguascalientes" },
+  BCN: { stateCode: "2", stateName: "Baja California" },
+  BCS: { stateCode: "3", stateName: "Baja California Sur" },
+  CAM: { stateCode: "4", stateName: "Campeche" },
+  CHP: { stateCode: "5", stateName: "Chiapas" },
+  CHH: { stateCode: "6", stateName: "Chihuahua" },
+  COA: { stateCode: "7", stateName: "Coahuila" },
+  COL: { stateCode: "8", stateName: "Colima" },
+  CMX: { stateCode: "9", stateName: "Ciudad de Mexico" },
+  CDMX: { stateCode: "9", stateName: "Ciudad de Mexico" },
+  DUR: { stateCode: "10", stateName: "Durango" },
+  GUA: { stateCode: "11", stateName: "Guanajuato" },
+  GRO: { stateCode: "12", stateName: "Guerrero" },
+  HID: { stateCode: "13", stateName: "Hidalgo" },
+  JAL: { stateCode: "14", stateName: "Jalisco" },
+  MEX: { stateCode: "15", stateName: "Estado de Mexico" },
+  MIC: { stateCode: "16", stateName: "Michoacan" },
+  MOR: { stateCode: "17", stateName: "Morelos" },
+  NAY: { stateCode: "18", stateName: "Nayarit" },
+  NLE: { stateCode: "19", stateName: "Nuevo Leon" },
+  OAX: { stateCode: "20", stateName: "Oaxaca" },
+  PUE: { stateCode: "21", stateName: "Puebla" },
+  QUE: { stateCode: "22", stateName: "Queretaro" },
+  ROO: { stateCode: "23", stateName: "Quintana Roo" },
+  SLP: { stateCode: "24", stateName: "San Luis Potosi" },
+  SIN: { stateCode: "25", stateName: "Sinaloa" },
+  SON: { stateCode: "26", stateName: "Sonora" },
+  TAB: { stateCode: "27", stateName: "Tabasco" },
+  TAM: { stateCode: "28", stateName: "Tamaulipas" },
+  TLA: { stateCode: "29", stateName: "Tlaxcala" },
+  VER: { stateCode: "30", stateName: "Veracruz" },
+  YUC: { stateCode: "31", stateName: "Yucatan" },
+  ZAC: { stateCode: "32", stateName: "Zacatecas" },
+};
+
+function loadDotenv(filePath) {
+  if (!existsSync(filePath)) {
+    return;
+  }
+
+  const raw = readFileSync(filePath, "utf8");
+
+  for (const line of raw.split(/\r?\n/)) {
+    const trimmed = line.trim();
+
+    if (!trimmed || trimmed.startsWith("#")) {
+      continue;
+    }
+
+    const eqIndex = trimmed.indexOf("=");
+    if (eqIndex === -1) {
+      continue;
+    }
+
+    const key = trimmed.slice(0, eqIndex).trim();
+
+    if (!key || process.env[key] !== undefined) {
+      continue;
+    }
+
+    let value = trimmed.slice(eqIndex + 1).trim();
+
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+      value = value.slice(1, -1);
+    }
+
+    process.env[key] = value;
+  }
+}
+
+function parseInteger(value, fallback) {
+  const parsed = Number.parseInt(value ?? "", 10);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function parseArgs(argv) {
+  const args = {
+    days: parseInteger(process.env.LICITAYA_BACKFILL_DAYS, DEFAULT_DAYS),
+    items: parseInteger(process.env.LICITAYA_BACKFILL_ITEMS, DEFAULT_ITEMS),
+    maxPages: parseInteger(process.env.LICITAYA_BACKFILL_MAX_PAGES, DEFAULT_MAX_PAGES),
+    delayMs: parseInteger(process.env.LICITAYA_BACKFILL_DELAY_MS, DEFAULT_DELAY_MS),
+    from: process.env.LICITAYA_BACKFILL_FROM || null,
+    to: process.env.LICITAYA_BACKFILL_TO || null,
+    dryRun: false,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const current = argv[index];
+    const next = argv[index + 1];
+
+    if (current === "--days" && next) {
+      args.days = parseInteger(next, args.days);
+      index += 1;
+      continue;
+    }
+
+    if (current === "--items" && next) {
+      args.items = parseInteger(next, args.items);
+      index += 1;
+      continue;
+    }
+
+    if (current === "--max-pages" && next) {
+      args.maxPages = parseInteger(next, args.maxPages);
+      index += 1;
+      continue;
+    }
+
+    if (current === "--delay-ms" && next) {
+      args.delayMs = parseInteger(next, args.delayMs);
+      index += 1;
+      continue;
+    }
+
+    if (current === "--from" && next) {
+      args.from = next;
+      index += 1;
+      continue;
+    }
+
+    if (current === "--to" && next) {
+      args.to = next;
+      index += 1;
+      continue;
+    }
+
+    if (current === "--dry-run") {
+      args.dryRun = true;
+      continue;
+    }
+  }
+
+  args.days = Math.max(1, args.days);
+  args.items = Math.min(100, Math.max(1, args.items));
+  args.maxPages = Math.max(1, args.maxPages);
+  args.delayMs = Math.max(0, args.delayMs);
+
+  return args;
+}
+
+function parseDateYmd(value) {
+  if (!value || !/^\d{8}$/.test(value)) {
+    return null;
+  }
+
+  const year = Number.parseInt(value.slice(0, 4), 10);
+  const month = Number.parseInt(value.slice(4, 6), 10);
+  const day = Number.parseInt(value.slice(6, 8), 10);
+  const date = new Date(Date.UTC(year, month - 1, day));
+  return Number.isNaN(date.getTime()) ? null : date;
+}
+
+function formatDateYmd(date) {
+  const year = String(date.getUTCFullYear());
+  const month = String(date.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(date.getUTCDate()).padStart(2, "0");
+  return `${year}${month}${day}`;
+}
+
+function buildDateRange(args) {
+  const toDate = parseDateYmd(args.to) ?? new Date();
+
+  if (args.from) {
+    const fromDate = parseDateYmd(args.from);
+
+    if (!fromDate) {
+      throw new Error("Invalid --from date. Use YYYYMMDD.");
+    }
+
+    const dates = [];
+    const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate()));
+    const floor = new Date(Date.UTC(fromDate.getUTCFullYear(), fromDate.getUTCMonth(), fromDate.getUTCDate()));
+
+    while (cursor >= floor) {
+      dates.push(formatDateYmd(cursor));
+      cursor.setUTCDate(cursor.getUTCDate() - 1);
+    }
+
+    return dates;
+  }
+
+  const dates = [];
+  const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate()));
+
+  for (let i = 0; i < args.days; i += 1) {
+    dates.push(formatDateYmd(cursor));
+    cursor.setUTCDate(cursor.getUTCDate() - 1);
+  }
+
+  return dates;
+}
+
+function toText(value) {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    return trimmed.length ? trimmed : null;
+  }
+
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+
+  return null;
+}
+
+function parseDateMaybe(value) {
+  const raw = toText(value);
+
+  if (!raw) {
+    return null;
+  }
+
+  const parsed = new Date(raw);
+  return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+
+function mapProcedureType(value) {
+  const text = (toText(value) || "").toLowerCase();
+
+  if (text.includes("adjudicacion directa")) {
+    return LicitationProcedureType.ADJUDICACION_DIRECTA;
+  }
+
+  if (text.includes("invitacion restringida") || text.includes("invitacion a cuando menos")) {
+    return LicitationProcedureType.INVITACION_RESTRINGIDA;
+  }
+
+  if (text.includes("licitacion publica") || text.includes("licitacion")) {
+    return LicitationProcedureType.LICITACION_PUBLICA;
+  }
+
+  return LicitationProcedureType.UNKNOWN;
+}
+
+function mapCategory(title, description) {
+  const text = `${title || ""} ${description || ""}`.toLowerCase();
+  const goods = text.includes("bien") || text.includes("insumo") || text.includes("material");
+  const services = text.includes("servicio") || text.includes("consultoria");
+  const works = text.includes("obra") || text.includes("construccion") || text.includes("infraestructura");
+  const matched = [goods, services, works].filter(Boolean).length;
+
+  if (matched > 1) {
+    return LicitationCategory.MIXED;
+  }
+
+  if (goods) {
+    return LicitationCategory.GOODS;
+  }
+
+  if (services) {
+    return LicitationCategory.SERVICES;
+  }
+
+  if (works) {
+    return LicitationCategory.WORKS;
+  }
+
+  return LicitationCategory.UNKNOWN;
+}
+
+function sanitizeDocuments(record) {
+  const docs = [];
+
+  if (toText(record.url)) {
+    docs.push({ name: "Ficha LicitaYa", url: String(record.url) });
+  }
+
+  if (toText(record.url2)) {
+    docs.push({ name: "Fuente alterna", url: String(record.url2) });
+  }
+
+  const seen = new Set();
+
+  return docs.filter((doc) => {
+    if (seen.has(doc.url)) {
+      return false;
+    }
+
+    seen.add(doc.url);
+    return true;
+  });
+}
+
+function resolveLicitayaState(rawState, stateNamesByCode) {
+  const token = (toText(rawState) || "").toUpperCase().replace(/[^A-Z]/g, "");
+  const mapped = LICITAYA_STATE_MAP[token] ?? null;
+
+  if (!mapped) {
+    return {
+      stateCode: "00",
+      stateName: "Cobertura nacional",
+    };
+  }
+
+  return {
+    stateCode: mapped.stateCode,
+    stateName: stateNamesByCode.get(mapped.stateCode) || mapped.stateName,
+  };
+}
+
+async function delay(ms) {
+  if (ms <= 0) {
+    return;
+  }
+
+  await new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
+}
+
+async function main() {
+  loadDotenv(resolve(process.cwd(), ".env"));
+
+  const args = parseArgs(process.argv.slice(2));
+  const dates = buildDateRange(args);
+  const apiKey = process.env.LICITAYA_API_KEY?.trim();
+  const baseUrl = (process.env.LICITAYA_BASE_URL?.trim() || "https://www.licitaya.com.mx/api/v1").replace(/\/+$/, "");
+
+  if (!apiKey) {
+    throw new Error("LICITAYA_API_KEY is required.");
+  }
+
+  const prisma = new PrismaClient();
+
+  try {
+    const stateRows = await prisma.municipality.findMany({
+      where: {
+        isActive: true,
+        municipalityCode: {
+          not: LICITAYA_MUNICIPALITY_CODE,
+        },
+      },
+      select: {
+        stateCode: true,
+        stateName: true,
+      },
+      distinct: ["stateCode"],
+    });
+
+    const stateNamesByCode = new Map(stateRows.map((row) => [row.stateCode, row.stateName]));
+    const municipalityCache = new Map();
+
+    async function ensureLicitayaMunicipality(stateCode, stateName) {
+      const key = `${stateCode}|${stateName}`;
+
+      if (municipalityCache.has(key)) {
+        return municipalityCache.get(key);
+      }
+
+      const municipality = await prisma.municipality.upsert({
+        where: {
+          stateCode_municipalityCode: {
+            stateCode,
+            municipalityCode: LICITAYA_MUNICIPALITY_CODE,
+          },
+        },
+        update: {
+          stateName,
+          municipalityName: `Cobertura LicitaYa (${stateName})`,
+          isActive: true,
+          scrapingEnabled: false,
+          openPortalType: MunicipalOpenPortalType.GENERIC,
+          openPortalUrl: null,
+          backupUrl: null,
+          pntEntryUrl: null,
+          pntEntityId: null,
+          pntSectorId: null,
+          pntSubjectId: null,
+        },
+        create: {
+          stateCode,
+          stateName,
+          municipalityCode: LICITAYA_MUNICIPALITY_CODE,
+          municipalityName: `Cobertura LicitaYa (${stateName})`,
+          openPortalType: MunicipalOpenPortalType.GENERIC,
+          isActive: true,
+          scrapingEnabled: false,
+        },
+        select: {
+          id: true,
+        },
+      });
+
+      municipalityCache.set(key, municipality);
+      return municipality;
+    }
+
+    const summary = {
+      datesAttempted: dates.length,
+      pagesAttempted: 0,
+      emptyDates: 0,
+      fetched: 0,
+      inserted: 0,
+      updated: 0,
+      skipped: 0,
+      errors: 0,
+      warnings: [],
+      dryRun: args.dryRun,
+    };
+
+    for (const date of dates) {
+      let dateHadResults = false;
+
+      for (let page = 1; page <= args.maxPages; page += 1) {
+        const requestUrl = new URL("tender/search", `${baseUrl}/`);
+        requestUrl.searchParams.set("date", date);
+        requestUrl.searchParams.set("items", String(args.items));
+        requestUrl.searchParams.set("page", String(page));
+        requestUrl.searchParams.set("order", "1");
+
+        summary.pagesAttempted += 1;
+
+        const response = await fetch(requestUrl, {
+          headers: {
+            "X-API-KEY": apiKey,
+            Accept: "application/json",
+          },
+        });
+
+        if (response.status === 404) {
+          break;
+        }
+
+        if (!response.ok) {
+          summary.errors += 1;
+          summary.warnings.push(`HTTP ${response.status} for ${requestUrl.toString()}`);
+          break;
+        }
+
+        const payload = (await response.json().catch(() => null));
+
+        if (!payload || typeof payload !== "object") {
+          summary.errors += 1;
+          summary.warnings.push(`Invalid JSON for ${requestUrl.toString()}`);
+          break;
+        }
+
+        const rows = Array.isArray(payload.results) ? payload.results : [];
+
+        if (!rows.length) {
+          break;
+        }
+
+        dateHadResults = true;
+
+        for (const row of rows) {
+          if (!row || typeof row !== "object") {
+            continue;
+          }
+
+          const record = row;
+          const tenderId = toText(record.tenderId);
+          const title = toText(record.tender_object) || toText(record.expanded_search) || (tenderId ? `Licitacion ${tenderId}` : null);
+
+          if (!tenderId || !title) {
+            summary.skipped += 1;
+            continue;
+          }
+
+          summary.fetched += 1;
+
+          const description = toText(record.expanded_search) || toText(record.extra_info) || null;
+          const publishDate = parseDateMaybe(record.catalog_date);
+          const closingDate = parseDateMaybe(record.close_date);
+          const state = resolveLicitayaState(record.state, stateNamesByCode);
+          const municipality = await ensureLicitayaMunicipality(state.stateCode, state.stateName);
+
+          if (args.dryRun) {
+            continue;
+          }
+
+          const sourceRecordId = tenderId.slice(0, 255);
+          const existing = await prisma.licitation.findUnique({
+            where: {
+              municipalityId_source_sourceRecordId: {
+                municipalityId: municipality.id,
+                source: LicitationSource.LICITAYA,
+                sourceRecordId,
+              },
+            },
+            select: {
+              id: true,
+            },
+          });
+
+          await prisma.licitation.upsert({
+            where: {
+              municipalityId_source_sourceRecordId: {
+                municipalityId: municipality.id,
+                source: LicitationSource.LICITAYA,
+                sourceRecordId,
+              },
+            },
+            update: {
+              tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null,
+              procedureType: mapProcedureType(record.type),
+              title,
+              description,
+              category: mapCategory(title, description),
+              isOpen: closingDate ? closingDate >= new Date() : true,
+              closingDate,
+              publishDate,
+              supplierAwarded: toText(record.agency),
+              documents: sanitizeDocuments(record),
+              rawSourceUrl: requestUrl.toString(),
+              rawPayload: {
+                source: "licitaya-backfill",
+                licitayaState: toText(record.state),
+                licitayaCity: toText(record.city),
+                record,
+              },
+              lastSeenAt: new Date(),
+            },
+            create: {
+              municipalityId: municipality.id,
+              source: LicitationSource.LICITAYA,
+              sourceRecordId,
+              tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null,
+              procedureType: mapProcedureType(record.type),
+              title,
+              description,
+              category: mapCategory(title, description),
+              isOpen: closingDate ? closingDate >= new Date() : true,
+              closingDate,
+              publishDate,
+              supplierAwarded: toText(record.agency),
+              documents: sanitizeDocuments(record),
+              rawSourceUrl: requestUrl.toString(),
+              rawPayload: {
+                source: "licitaya-backfill",
+                licitayaState: toText(record.state),
+                licitayaCity: toText(record.city),
+                record,
+              },
+              lastSeenAt: new Date(),
+            },
+          });
+
+          if (existing) {
+            summary.updated += 1;
+          } else {
+            summary.inserted += 1;
+          }
+        }
+
+        const totalPages = Number.parseInt(toText(payload.total_pages) || "1", 10);
+
+        if (!Number.isFinite(totalPages) || page >= totalPages || page >= args.maxPages) {
+          break;
+        }
+
+        await delay(args.delayMs);
+      }
+
+      if (!dateHadResults) {
+        summary.emptyDates += 1;
+      }
+    }
+
+    console.log(JSON.stringify(summary, null, 2));
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});

scripts/test-licitaya-api.mjs

@@ -2,6 +2,25 @@ import { readFileSync, existsSync } from "node:fs";
 import { resolve } from "node:path";
 
 const DEFAULT_TIMEOUT_MS = 20000;
+const DEFAULT_ENDPOINT = "/tender/SCRZJ";
+
+function parseBoolean(value, fallback = false) {
+  if (!value) {
+    return fallback;
+  }
+
+  const normalized = value.trim().toLowerCase();
+
+  if (["1", "true", "yes", "si"].includes(normalized)) {
+    return true;
+  }
+
+  if (["0", "false", "no"].includes(normalized)) {
+    return false;
+  }
+
+  return fallback;
+}
 
 function loadDotenv(filePath) {
   if (!existsSync(filePath)) {
@@ -40,10 +59,11 @@ function loadDotenv(filePath) {
 function parseArgs(argv) {
   const args = {
     baseUrl: process.env.LICITAYA_BASE_URL,
-    endpoint: process.env.LICITAYA_TEST_ENDPOINT,
+    endpoint: process.env.LICITAYA_TEST_ENDPOINT || DEFAULT_ENDPOINT,
     accept: process.env.LICITAYA_ACCEPT || "application/json",
     method: "GET",
     timeoutMs: Number.parseInt(process.env.LICITAYA_TIMEOUT_MS || "", 10) || DEFAULT_TIMEOUT_MS,
+    allowEmptySearch: parseBoolean(process.env.LICITAYA_ALLOW_EMPTY_SEARCH, true),
   };
 
   for (let i = 0; i < argv.length; i += 1) {
@@ -82,6 +102,11 @@ function parseArgs(argv) {
       i += 1;
       continue;
     }
+
+    if (current === "--allow-empty-search") {
+      args.allowEmptySearch = true;
+      continue;
+    }
   }
 
   return args;
@@ -176,10 +201,17 @@ try {
   console.log("--- Response Preview ---");
   console.log(bodyPreview);
 
-  if (response.status === 404 && url.pathname.endsWith("/tender/search")) {
+  const isEmptySearch404 = response.status === 404 && url.pathname.endsWith("/tender/search");
+
+  if (isEmptySearch404) {
     console.error("No tenders matched the current filters. Try a broader keyword or fewer filters.");
   }
 
+  if (isEmptySearch404 && args.allowEmptySearch) {
+    console.log("Connectivity check passed (search endpoint returned 404 with no matches).");
+    process.exit(0);
+  }
+
   if (!response.ok) {
     process.exit(1);
   }