import { existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; import { PrismaClient, LicitationCategory, LicitationProcedureType, LicitationSource, MunicipalOpenPortalType } from "@prisma/client"; const DEFAULT_ITEMS = 50; const DEFAULT_MAX_PAGES = 5; const DEFAULT_DAYS = 30; const DEFAULT_DELAY_MS = 400; const LICITAYA_MUNICIPALITY_CODE = "LICITAYA"; const LICITAYA_STATE_MAP = { AGS: { stateCode: "1", stateName: "Aguascalientes" }, BCN: { stateCode: "2", stateName: "Baja California" }, BCS: { stateCode: "3", stateName: "Baja California Sur" }, CAM: { stateCode: "4", stateName: "Campeche" }, CHP: { stateCode: "5", stateName: "Chiapas" }, CHH: { stateCode: "6", stateName: "Chihuahua" }, COA: { stateCode: "7", stateName: "Coahuila" }, COL: { stateCode: "8", stateName: "Colima" }, CMX: { stateCode: "9", stateName: "Ciudad de Mexico" }, CDMX: { stateCode: "9", stateName: "Ciudad de Mexico" }, DUR: { stateCode: "10", stateName: "Durango" }, GUA: { stateCode: "11", stateName: "Guanajuato" }, GRO: { stateCode: "12", stateName: "Guerrero" }, HID: { stateCode: "13", stateName: "Hidalgo" }, JAL: { stateCode: "14", stateName: "Jalisco" }, MEX: { stateCode: "15", stateName: "Estado de Mexico" }, MIC: { stateCode: "16", stateName: "Michoacan" }, MOR: { stateCode: "17", stateName: "Morelos" }, NAY: { stateCode: "18", stateName: "Nayarit" }, NLE: { stateCode: "19", stateName: "Nuevo Leon" }, OAX: { stateCode: "20", stateName: "Oaxaca" }, PUE: { stateCode: "21", stateName: "Puebla" }, QUE: { stateCode: "22", stateName: "Queretaro" }, ROO: { stateCode: "23", stateName: "Quintana Roo" }, SLP: { stateCode: "24", stateName: "San Luis Potosi" }, SIN: { stateCode: "25", stateName: "Sinaloa" }, SON: { stateCode: "26", stateName: "Sonora" }, TAB: { stateCode: "27", stateName: "Tabasco" }, TAM: { stateCode: "28", stateName: "Tamaulipas" }, TLA: { stateCode: "29", stateName: "Tlaxcala" }, VER: { stateCode: "30", stateName: "Veracruz" }, YUC: { stateCode: "31", stateName: "Yucatan" }, ZAC: { stateCode: "32", stateName: "Zacatecas" }, }; function loadDotenv(filePath) { if (!existsSync(filePath)) { return; } const raw = readFileSync(filePath, "utf8"); for (const line of raw.split(/\r?\n/)) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) { continue; } const eqIndex = trimmed.indexOf("="); if (eqIndex === -1) { continue; } const key = trimmed.slice(0, eqIndex).trim(); if (!key || process.env[key] !== undefined) { continue; } let value = trimmed.slice(eqIndex + 1).trim(); if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { value = value.slice(1, -1); } process.env[key] = value; } } function parseInteger(value, fallback) { const parsed = Number.parseInt(value ?? "", 10); return Number.isFinite(parsed) ? parsed : fallback; } function parseArgs(argv) { const args = { days: parseInteger(process.env.LICITAYA_BACKFILL_DAYS, DEFAULT_DAYS), items: parseInteger(process.env.LICITAYA_BACKFILL_ITEMS, DEFAULT_ITEMS), maxPages: parseInteger(process.env.LICITAYA_BACKFILL_MAX_PAGES, DEFAULT_MAX_PAGES), delayMs: parseInteger(process.env.LICITAYA_BACKFILL_DELAY_MS, DEFAULT_DELAY_MS), from: process.env.LICITAYA_BACKFILL_FROM || null, to: process.env.LICITAYA_BACKFILL_TO || null, dryRun: false, }; for (let index = 0; index < argv.length; index += 1) { const current = argv[index]; const next = argv[index + 1]; if (current === "--days" && next) { args.days = parseInteger(next, args.days); index += 1; continue; } if (current === "--items" && next) { args.items = parseInteger(next, args.items); index += 1; continue; } if (current === "--max-pages" && next) { args.maxPages = parseInteger(next, args.maxPages); index += 1; continue; } if (current === "--delay-ms" && next) { args.delayMs = parseInteger(next, args.delayMs); index += 1; continue; } if (current === "--from" && next) { args.from = next; index += 1; continue; } if (current === "--to" && next) { args.to = next; index += 1; continue; } if (current === "--dry-run") { args.dryRun = true; continue; } } args.days = Math.max(1, args.days); args.items = Math.min(100, Math.max(1, args.items)); args.maxPages = Math.max(1, args.maxPages); args.delayMs = Math.max(0, args.delayMs); return args; } function parseDateYmd(value) { if (!value || !/^\d{8}$/.test(value)) { return null; } const year = Number.parseInt(value.slice(0, 4), 10); const month = Number.parseInt(value.slice(4, 6), 10); const day = Number.parseInt(value.slice(6, 8), 10); const date = new Date(Date.UTC(year, month - 1, day)); return Number.isNaN(date.getTime()) ? null : date; } function formatDateYmd(date) { const year = String(date.getUTCFullYear()); const month = String(date.getUTCMonth() + 1).padStart(2, "0"); const day = String(date.getUTCDate()).padStart(2, "0"); return `${year}${month}${day}`; } function buildDateRange(args) { const toDate = parseDateYmd(args.to) ?? new Date(); if (args.from) { const fromDate = parseDateYmd(args.from); if (!fromDate) { throw new Error("Invalid --from date. Use YYYYMMDD."); } const dates = []; const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate())); const floor = new Date(Date.UTC(fromDate.getUTCFullYear(), fromDate.getUTCMonth(), fromDate.getUTCDate())); while (cursor >= floor) { dates.push(formatDateYmd(cursor)); cursor.setUTCDate(cursor.getUTCDate() - 1); } return dates; } const dates = []; const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate())); for (let i = 0; i < args.days; i += 1) { dates.push(formatDateYmd(cursor)); cursor.setUTCDate(cursor.getUTCDate() - 1); } return dates; } function toText(value) { if (typeof value === "string") { const trimmed = value.trim(); return trimmed.length ? trimmed : null; } if (typeof value === "number" || typeof value === "boolean") { return String(value); } return null; } function parseDateMaybe(value) { const raw = toText(value); if (!raw) { return null; } const parsed = new Date(raw); return Number.isNaN(parsed.getTime()) ? null : parsed; } function mapProcedureType(value) { const text = (toText(value) || "").toLowerCase(); if (text.includes("adjudicacion directa")) { return LicitationProcedureType.ADJUDICACION_DIRECTA; } if (text.includes("invitacion restringida") || text.includes("invitacion a cuando menos")) { return LicitationProcedureType.INVITACION_RESTRINGIDA; } if (text.includes("licitacion publica") || text.includes("licitacion")) { return LicitationProcedureType.LICITACION_PUBLICA; } return LicitationProcedureType.UNKNOWN; } function mapCategory(title, description) { const text = `${title || ""} ${description || ""}`.toLowerCase(); const goods = text.includes("bien") || text.includes("insumo") || text.includes("material"); const services = text.includes("servicio") || text.includes("consultoria"); const works = text.includes("obra") || text.includes("construccion") || text.includes("infraestructura"); const matched = [goods, services, works].filter(Boolean).length; if (matched > 1) { return LicitationCategory.MIXED; } if (goods) { return LicitationCategory.GOODS; } if (services) { return LicitationCategory.SERVICES; } if (works) { return LicitationCategory.WORKS; } return LicitationCategory.UNKNOWN; } function sanitizeDocuments(record) { const docs = []; if (toText(record.url)) { docs.push({ name: "Ficha LicitaYa", url: String(record.url) }); } if (toText(record.url2)) { docs.push({ name: "Fuente alterna", url: String(record.url2) }); } const seen = new Set(); return docs.filter((doc) => { if (seen.has(doc.url)) { return false; } seen.add(doc.url); return true; }); } function resolveLicitayaState(rawState, stateNamesByCode) { const token = (toText(rawState) || "").toUpperCase().replace(/[^A-Z]/g, ""); const mapped = LICITAYA_STATE_MAP[token] ?? null; if (!mapped) { return { stateCode: "00", stateName: "Cobertura nacional", }; } return { stateCode: mapped.stateCode, stateName: stateNamesByCode.get(mapped.stateCode) || mapped.stateName, }; } async function delay(ms) { if (ms <= 0) { return; } await new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); } async function main() { loadDotenv(resolve(process.cwd(), ".env")); const args = parseArgs(process.argv.slice(2)); const dates = buildDateRange(args); const apiKey = process.env.LICITAYA_API_KEY?.trim(); const baseUrl = (process.env.LICITAYA_BASE_URL?.trim() || "https://www.licitaya.com.mx/api/v1").replace(/\/+$/, ""); if (!apiKey) { throw new Error("LICITAYA_API_KEY is required."); } const prisma = new PrismaClient(); try { const stateRows = await prisma.municipality.findMany({ where: { isActive: true, municipalityCode: { not: LICITAYA_MUNICIPALITY_CODE, }, }, select: { stateCode: true, stateName: true, }, distinct: ["stateCode"], }); const stateNamesByCode = new Map(stateRows.map((row) => [row.stateCode, row.stateName])); const municipalityCache = new Map(); async function ensureLicitayaMunicipality(stateCode, stateName) { const key = `${stateCode}|${stateName}`; if (municipalityCache.has(key)) { return municipalityCache.get(key); } const municipality = await prisma.municipality.upsert({ where: { stateCode_municipalityCode: { stateCode, municipalityCode: LICITAYA_MUNICIPALITY_CODE, }, }, update: { stateName, municipalityName: `Cobertura LicitaYa (${stateName})`, isActive: true, scrapingEnabled: false, openPortalType: MunicipalOpenPortalType.GENERIC, openPortalUrl: null, backupUrl: null, pntEntryUrl: null, pntEntityId: null, pntSectorId: null, pntSubjectId: null, }, create: { stateCode, stateName, municipalityCode: LICITAYA_MUNICIPALITY_CODE, municipalityName: `Cobertura LicitaYa (${stateName})`, openPortalType: MunicipalOpenPortalType.GENERIC, isActive: true, scrapingEnabled: false, }, select: { id: true, }, }); municipalityCache.set(key, municipality); return municipality; } const summary = { datesAttempted: dates.length, pagesAttempted: 0, emptyDates: 0, fetched: 0, inserted: 0, updated: 0, skipped: 0, errors: 0, warnings: [], dryRun: args.dryRun, }; for (const date of dates) { let dateHadResults = false; for (let page = 1; page <= args.maxPages; page += 1) { const requestUrl = new URL("tender/search", `${baseUrl}/`); requestUrl.searchParams.set("date", date); requestUrl.searchParams.set("items", String(args.items)); requestUrl.searchParams.set("page", String(page)); requestUrl.searchParams.set("order", "1"); summary.pagesAttempted += 1; const response = await fetch(requestUrl, { headers: { "X-API-KEY": apiKey, Accept: "application/json", }, }); if (response.status === 404) { break; } if (!response.ok) { summary.errors += 1; summary.warnings.push(`HTTP ${response.status} for ${requestUrl.toString()}`); break; } const payload = (await response.json().catch(() => null)); if (!payload || typeof payload !== "object") { summary.errors += 1; summary.warnings.push(`Invalid JSON for ${requestUrl.toString()}`); break; } const rows = Array.isArray(payload.results) ? payload.results : []; if (!rows.length) { break; } dateHadResults = true; for (const row of rows) { if (!row || typeof row !== "object") { continue; } const record = row; const tenderId = toText(record.tenderId); const title = toText(record.tender_object) || toText(record.expanded_search) || (tenderId ? `Licitacion ${tenderId}` : null); if (!tenderId || !title) { summary.skipped += 1; continue; } summary.fetched += 1; const description = toText(record.expanded_search) || toText(record.extra_info) || null; const publishDate = parseDateMaybe(record.catalog_date); const closingDate = parseDateMaybe(record.close_date); const state = resolveLicitayaState(record.state, stateNamesByCode); const municipality = await ensureLicitayaMunicipality(state.stateCode, state.stateName); if (args.dryRun) { continue; } const sourceRecordId = tenderId.slice(0, 255); const existing = await prisma.licitation.findUnique({ where: { municipalityId_source_sourceRecordId: { municipalityId: municipality.id, source: LicitationSource.LICITAYA, sourceRecordId, }, }, select: { id: true, }, }); await prisma.licitation.upsert({ where: { municipalityId_source_sourceRecordId: { municipalityId: municipality.id, source: LicitationSource.LICITAYA, sourceRecordId, }, }, update: { tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null, procedureType: mapProcedureType(record.type), title, description, category: mapCategory(title, description), isOpen: closingDate ? closingDate >= new Date() : true, closingDate, publishDate, supplierAwarded: toText(record.agency), documents: sanitizeDocuments(record), rawSourceUrl: requestUrl.toString(), rawPayload: { source: "licitaya-backfill", licitayaState: toText(record.state), licitayaCity: toText(record.city), record, }, lastSeenAt: new Date(), }, create: { municipalityId: municipality.id, source: LicitationSource.LICITAYA, sourceRecordId, tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null, procedureType: mapProcedureType(record.type), title, description, category: mapCategory(title, description), isOpen: closingDate ? closingDate >= new Date() : true, closingDate, publishDate, supplierAwarded: toText(record.agency), documents: sanitizeDocuments(record), rawSourceUrl: requestUrl.toString(), rawPayload: { source: "licitaya-backfill", licitayaState: toText(record.state), licitayaCity: toText(record.city), record, }, lastSeenAt: new Date(), }, }); if (existing) { summary.updated += 1; } else { summary.inserted += 1; } } const totalPages = Number.parseInt(toText(payload.total_pages) || "1", 10); if (!Number.isFinite(totalPages) || page >= totalPages || page >= args.maxPages) { break; } await delay(args.delayMs); } if (!dateHadResults) { summary.emptyDates += 1; } } console.log(JSON.stringify(summary, null, 2)); } finally { await prisma.$disconnect(); } } main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); });