579 lines
16 KiB
JavaScript
579 lines
16 KiB
JavaScript
import { existsSync, readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { PrismaClient, LicitationCategory, LicitationProcedureType, LicitationSource, MunicipalOpenPortalType } from "@prisma/client";
|
|
|
|
const DEFAULT_ITEMS = 50;
|
|
const DEFAULT_MAX_PAGES = 5;
|
|
const DEFAULT_DAYS = 30;
|
|
const DEFAULT_DELAY_MS = 400;
|
|
const LICITAYA_MUNICIPALITY_CODE = "LICITAYA";
|
|
|
|
const LICITAYA_STATE_MAP = {
|
|
AGS: { stateCode: "1", stateName: "Aguascalientes" },
|
|
BCN: { stateCode: "2", stateName: "Baja California" },
|
|
BCS: { stateCode: "3", stateName: "Baja California Sur" },
|
|
CAM: { stateCode: "4", stateName: "Campeche" },
|
|
CHP: { stateCode: "5", stateName: "Chiapas" },
|
|
CHH: { stateCode: "6", stateName: "Chihuahua" },
|
|
COA: { stateCode: "7", stateName: "Coahuila" },
|
|
COL: { stateCode: "8", stateName: "Colima" },
|
|
CMX: { stateCode: "9", stateName: "Ciudad de Mexico" },
|
|
CDMX: { stateCode: "9", stateName: "Ciudad de Mexico" },
|
|
DUR: { stateCode: "10", stateName: "Durango" },
|
|
GUA: { stateCode: "11", stateName: "Guanajuato" },
|
|
GRO: { stateCode: "12", stateName: "Guerrero" },
|
|
HID: { stateCode: "13", stateName: "Hidalgo" },
|
|
JAL: { stateCode: "14", stateName: "Jalisco" },
|
|
MEX: { stateCode: "15", stateName: "Estado de Mexico" },
|
|
MIC: { stateCode: "16", stateName: "Michoacan" },
|
|
MOR: { stateCode: "17", stateName: "Morelos" },
|
|
NAY: { stateCode: "18", stateName: "Nayarit" },
|
|
NLE: { stateCode: "19", stateName: "Nuevo Leon" },
|
|
OAX: { stateCode: "20", stateName: "Oaxaca" },
|
|
PUE: { stateCode: "21", stateName: "Puebla" },
|
|
QUE: { stateCode: "22", stateName: "Queretaro" },
|
|
ROO: { stateCode: "23", stateName: "Quintana Roo" },
|
|
SLP: { stateCode: "24", stateName: "San Luis Potosi" },
|
|
SIN: { stateCode: "25", stateName: "Sinaloa" },
|
|
SON: { stateCode: "26", stateName: "Sonora" },
|
|
TAB: { stateCode: "27", stateName: "Tabasco" },
|
|
TAM: { stateCode: "28", stateName: "Tamaulipas" },
|
|
TLA: { stateCode: "29", stateName: "Tlaxcala" },
|
|
VER: { stateCode: "30", stateName: "Veracruz" },
|
|
YUC: { stateCode: "31", stateName: "Yucatan" },
|
|
ZAC: { stateCode: "32", stateName: "Zacatecas" },
|
|
};
|
|
|
|
function loadDotenv(filePath) {
|
|
if (!existsSync(filePath)) {
|
|
return;
|
|
}
|
|
|
|
const raw = readFileSync(filePath, "utf8");
|
|
|
|
for (const line of raw.split(/\r?\n/)) {
|
|
const trimmed = line.trim();
|
|
|
|
if (!trimmed || trimmed.startsWith("#")) {
|
|
continue;
|
|
}
|
|
|
|
const eqIndex = trimmed.indexOf("=");
|
|
if (eqIndex === -1) {
|
|
continue;
|
|
}
|
|
|
|
const key = trimmed.slice(0, eqIndex).trim();
|
|
|
|
if (!key || process.env[key] !== undefined) {
|
|
continue;
|
|
}
|
|
|
|
let value = trimmed.slice(eqIndex + 1).trim();
|
|
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
|
|
process.env[key] = value;
|
|
}
|
|
}
|
|
|
|
function parseInteger(value, fallback) {
|
|
const parsed = Number.parseInt(value ?? "", 10);
|
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
days: parseInteger(process.env.LICITAYA_BACKFILL_DAYS, DEFAULT_DAYS),
|
|
items: parseInteger(process.env.LICITAYA_BACKFILL_ITEMS, DEFAULT_ITEMS),
|
|
maxPages: parseInteger(process.env.LICITAYA_BACKFILL_MAX_PAGES, DEFAULT_MAX_PAGES),
|
|
delayMs: parseInteger(process.env.LICITAYA_BACKFILL_DELAY_MS, DEFAULT_DELAY_MS),
|
|
from: process.env.LICITAYA_BACKFILL_FROM || null,
|
|
to: process.env.LICITAYA_BACKFILL_TO || null,
|
|
dryRun: false,
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const current = argv[index];
|
|
const next = argv[index + 1];
|
|
|
|
if (current === "--days" && next) {
|
|
args.days = parseInteger(next, args.days);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--items" && next) {
|
|
args.items = parseInteger(next, args.items);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--max-pages" && next) {
|
|
args.maxPages = parseInteger(next, args.maxPages);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--delay-ms" && next) {
|
|
args.delayMs = parseInteger(next, args.delayMs);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--from" && next) {
|
|
args.from = next;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--to" && next) {
|
|
args.to = next;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
if (current === "--dry-run") {
|
|
args.dryRun = true;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
args.days = Math.max(1, args.days);
|
|
args.items = Math.min(100, Math.max(1, args.items));
|
|
args.maxPages = Math.max(1, args.maxPages);
|
|
args.delayMs = Math.max(0, args.delayMs);
|
|
|
|
return args;
|
|
}
|
|
|
|
function parseDateYmd(value) {
|
|
if (!value || !/^\d{8}$/.test(value)) {
|
|
return null;
|
|
}
|
|
|
|
const year = Number.parseInt(value.slice(0, 4), 10);
|
|
const month = Number.parseInt(value.slice(4, 6), 10);
|
|
const day = Number.parseInt(value.slice(6, 8), 10);
|
|
const date = new Date(Date.UTC(year, month - 1, day));
|
|
return Number.isNaN(date.getTime()) ? null : date;
|
|
}
|
|
|
|
function formatDateYmd(date) {
|
|
const year = String(date.getUTCFullYear());
|
|
const month = String(date.getUTCMonth() + 1).padStart(2, "0");
|
|
const day = String(date.getUTCDate()).padStart(2, "0");
|
|
return `${year}${month}${day}`;
|
|
}
|
|
|
|
function buildDateRange(args) {
|
|
const toDate = parseDateYmd(args.to) ?? new Date();
|
|
|
|
if (args.from) {
|
|
const fromDate = parseDateYmd(args.from);
|
|
|
|
if (!fromDate) {
|
|
throw new Error("Invalid --from date. Use YYYYMMDD.");
|
|
}
|
|
|
|
const dates = [];
|
|
const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate()));
|
|
const floor = new Date(Date.UTC(fromDate.getUTCFullYear(), fromDate.getUTCMonth(), fromDate.getUTCDate()));
|
|
|
|
while (cursor >= floor) {
|
|
dates.push(formatDateYmd(cursor));
|
|
cursor.setUTCDate(cursor.getUTCDate() - 1);
|
|
}
|
|
|
|
return dates;
|
|
}
|
|
|
|
const dates = [];
|
|
const cursor = new Date(Date.UTC(toDate.getUTCFullYear(), toDate.getUTCMonth(), toDate.getUTCDate()));
|
|
|
|
for (let i = 0; i < args.days; i += 1) {
|
|
dates.push(formatDateYmd(cursor));
|
|
cursor.setUTCDate(cursor.getUTCDate() - 1);
|
|
}
|
|
|
|
return dates;
|
|
}
|
|
|
|
function toText(value) {
|
|
if (typeof value === "string") {
|
|
const trimmed = value.trim();
|
|
return trimmed.length ? trimmed : null;
|
|
}
|
|
|
|
if (typeof value === "number" || typeof value === "boolean") {
|
|
return String(value);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function parseDateMaybe(value) {
|
|
const raw = toText(value);
|
|
|
|
if (!raw) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = new Date(raw);
|
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
}
|
|
|
|
function mapProcedureType(value) {
|
|
const text = (toText(value) || "").toLowerCase();
|
|
|
|
if (text.includes("adjudicacion directa")) {
|
|
return LicitationProcedureType.ADJUDICACION_DIRECTA;
|
|
}
|
|
|
|
if (text.includes("invitacion restringida") || text.includes("invitacion a cuando menos")) {
|
|
return LicitationProcedureType.INVITACION_RESTRINGIDA;
|
|
}
|
|
|
|
if (text.includes("licitacion publica") || text.includes("licitacion")) {
|
|
return LicitationProcedureType.LICITACION_PUBLICA;
|
|
}
|
|
|
|
return LicitationProcedureType.UNKNOWN;
|
|
}
|
|
|
|
function mapCategory(title, description) {
|
|
const text = `${title || ""} ${description || ""}`.toLowerCase();
|
|
const goods = text.includes("bien") || text.includes("insumo") || text.includes("material");
|
|
const services = text.includes("servicio") || text.includes("consultoria");
|
|
const works = text.includes("obra") || text.includes("construccion") || text.includes("infraestructura");
|
|
const matched = [goods, services, works].filter(Boolean).length;
|
|
|
|
if (matched > 1) {
|
|
return LicitationCategory.MIXED;
|
|
}
|
|
|
|
if (goods) {
|
|
return LicitationCategory.GOODS;
|
|
}
|
|
|
|
if (services) {
|
|
return LicitationCategory.SERVICES;
|
|
}
|
|
|
|
if (works) {
|
|
return LicitationCategory.WORKS;
|
|
}
|
|
|
|
return LicitationCategory.UNKNOWN;
|
|
}
|
|
|
|
function sanitizeDocuments(record) {
|
|
const docs = [];
|
|
|
|
if (toText(record.url)) {
|
|
docs.push({ name: "Ficha LicitaYa", url: String(record.url) });
|
|
}
|
|
|
|
if (toText(record.url2)) {
|
|
docs.push({ name: "Fuente alterna", url: String(record.url2) });
|
|
}
|
|
|
|
const seen = new Set();
|
|
|
|
return docs.filter((doc) => {
|
|
if (seen.has(doc.url)) {
|
|
return false;
|
|
}
|
|
|
|
seen.add(doc.url);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function resolveLicitayaState(rawState, stateNamesByCode) {
|
|
const token = (toText(rawState) || "").toUpperCase().replace(/[^A-Z]/g, "");
|
|
const mapped = LICITAYA_STATE_MAP[token] ?? null;
|
|
|
|
if (!mapped) {
|
|
return {
|
|
stateCode: "00",
|
|
stateName: "Cobertura nacional",
|
|
};
|
|
}
|
|
|
|
return {
|
|
stateCode: mapped.stateCode,
|
|
stateName: stateNamesByCode.get(mapped.stateCode) || mapped.stateName,
|
|
};
|
|
}
|
|
|
|
async function delay(ms) {
|
|
if (ms <= 0) {
|
|
return;
|
|
}
|
|
|
|
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
|
|
}
|
|
|
|
async function main() {
|
|
loadDotenv(resolve(process.cwd(), ".env"));
|
|
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const dates = buildDateRange(args);
|
|
const apiKey = process.env.LICITAYA_API_KEY?.trim();
|
|
const baseUrl = (process.env.LICITAYA_BASE_URL?.trim() || "https://www.licitaya.com.mx/api/v1").replace(/\/+$/, "");
|
|
|
|
if (!apiKey) {
|
|
throw new Error("LICITAYA_API_KEY is required.");
|
|
}
|
|
|
|
const prisma = new PrismaClient();
|
|
|
|
try {
|
|
const stateRows = await prisma.municipality.findMany({
|
|
where: {
|
|
isActive: true,
|
|
municipalityCode: {
|
|
not: LICITAYA_MUNICIPALITY_CODE,
|
|
},
|
|
},
|
|
select: {
|
|
stateCode: true,
|
|
stateName: true,
|
|
},
|
|
distinct: ["stateCode"],
|
|
});
|
|
|
|
const stateNamesByCode = new Map(stateRows.map((row) => [row.stateCode, row.stateName]));
|
|
const municipalityCache = new Map();
|
|
|
|
async function ensureLicitayaMunicipality(stateCode, stateName) {
|
|
const key = `${stateCode}|${stateName}`;
|
|
|
|
if (municipalityCache.has(key)) {
|
|
return municipalityCache.get(key);
|
|
}
|
|
|
|
const municipality = await prisma.municipality.upsert({
|
|
where: {
|
|
stateCode_municipalityCode: {
|
|
stateCode,
|
|
municipalityCode: LICITAYA_MUNICIPALITY_CODE,
|
|
},
|
|
},
|
|
update: {
|
|
stateName,
|
|
municipalityName: `Cobertura LicitaYa (${stateName})`,
|
|
isActive: true,
|
|
scrapingEnabled: false,
|
|
openPortalType: MunicipalOpenPortalType.GENERIC,
|
|
openPortalUrl: null,
|
|
backupUrl: null,
|
|
pntEntryUrl: null,
|
|
pntEntityId: null,
|
|
pntSectorId: null,
|
|
pntSubjectId: null,
|
|
},
|
|
create: {
|
|
stateCode,
|
|
stateName,
|
|
municipalityCode: LICITAYA_MUNICIPALITY_CODE,
|
|
municipalityName: `Cobertura LicitaYa (${stateName})`,
|
|
openPortalType: MunicipalOpenPortalType.GENERIC,
|
|
isActive: true,
|
|
scrapingEnabled: false,
|
|
},
|
|
select: {
|
|
id: true,
|
|
},
|
|
});
|
|
|
|
municipalityCache.set(key, municipality);
|
|
return municipality;
|
|
}
|
|
|
|
const summary = {
|
|
datesAttempted: dates.length,
|
|
pagesAttempted: 0,
|
|
emptyDates: 0,
|
|
fetched: 0,
|
|
inserted: 0,
|
|
updated: 0,
|
|
skipped: 0,
|
|
errors: 0,
|
|
warnings: [],
|
|
dryRun: args.dryRun,
|
|
};
|
|
|
|
for (const date of dates) {
|
|
let dateHadResults = false;
|
|
|
|
for (let page = 1; page <= args.maxPages; page += 1) {
|
|
const requestUrl = new URL("tender/search", `${baseUrl}/`);
|
|
requestUrl.searchParams.set("date", date);
|
|
requestUrl.searchParams.set("items", String(args.items));
|
|
requestUrl.searchParams.set("page", String(page));
|
|
requestUrl.searchParams.set("order", "1");
|
|
|
|
summary.pagesAttempted += 1;
|
|
|
|
const response = await fetch(requestUrl, {
|
|
headers: {
|
|
"X-API-KEY": apiKey,
|
|
Accept: "application/json",
|
|
},
|
|
});
|
|
|
|
if (response.status === 404) {
|
|
break;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
summary.errors += 1;
|
|
summary.warnings.push(`HTTP ${response.status} for ${requestUrl.toString()}`);
|
|
break;
|
|
}
|
|
|
|
const payload = (await response.json().catch(() => null));
|
|
|
|
if (!payload || typeof payload !== "object") {
|
|
summary.errors += 1;
|
|
summary.warnings.push(`Invalid JSON for ${requestUrl.toString()}`);
|
|
break;
|
|
}
|
|
|
|
const rows = Array.isArray(payload.results) ? payload.results : [];
|
|
|
|
if (!rows.length) {
|
|
break;
|
|
}
|
|
|
|
dateHadResults = true;
|
|
|
|
for (const row of rows) {
|
|
if (!row || typeof row !== "object") {
|
|
continue;
|
|
}
|
|
|
|
const record = row;
|
|
const tenderId = toText(record.tenderId);
|
|
const title = toText(record.tender_object) || toText(record.expanded_search) || (tenderId ? `Licitacion ${tenderId}` : null);
|
|
|
|
if (!tenderId || !title) {
|
|
summary.skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
summary.fetched += 1;
|
|
|
|
const description = toText(record.expanded_search) || toText(record.extra_info) || null;
|
|
const publishDate = parseDateMaybe(record.catalog_date);
|
|
const closingDate = parseDateMaybe(record.close_date);
|
|
const state = resolveLicitayaState(record.state, stateNamesByCode);
|
|
const municipality = await ensureLicitayaMunicipality(state.stateCode, state.stateName);
|
|
|
|
if (args.dryRun) {
|
|
continue;
|
|
}
|
|
|
|
const sourceRecordId = tenderId.slice(0, 255);
|
|
const existing = await prisma.licitation.findUnique({
|
|
where: {
|
|
municipalityId_source_sourceRecordId: {
|
|
municipalityId: municipality.id,
|
|
source: LicitationSource.LICITAYA,
|
|
sourceRecordId,
|
|
},
|
|
},
|
|
select: {
|
|
id: true,
|
|
},
|
|
});
|
|
|
|
await prisma.licitation.upsert({
|
|
where: {
|
|
municipalityId_source_sourceRecordId: {
|
|
municipalityId: municipality.id,
|
|
source: LicitationSource.LICITAYA,
|
|
sourceRecordId,
|
|
},
|
|
},
|
|
update: {
|
|
tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null,
|
|
procedureType: mapProcedureType(record.type),
|
|
title,
|
|
description,
|
|
category: mapCategory(title, description),
|
|
isOpen: closingDate ? closingDate >= new Date() : true,
|
|
closingDate,
|
|
publishDate,
|
|
supplierAwarded: toText(record.agency),
|
|
documents: sanitizeDocuments(record),
|
|
rawSourceUrl: requestUrl.toString(),
|
|
rawPayload: {
|
|
source: "licitaya-backfill",
|
|
licitayaState: toText(record.state),
|
|
licitayaCity: toText(record.city),
|
|
record,
|
|
},
|
|
lastSeenAt: new Date(),
|
|
},
|
|
create: {
|
|
municipalityId: municipality.id,
|
|
source: LicitationSource.LICITAYA,
|
|
sourceRecordId,
|
|
tenderCode: toText(record.number) || toText(record.number2) || toText(record.reference) || toText(record.process) || null,
|
|
procedureType: mapProcedureType(record.type),
|
|
title,
|
|
description,
|
|
category: mapCategory(title, description),
|
|
isOpen: closingDate ? closingDate >= new Date() : true,
|
|
closingDate,
|
|
publishDate,
|
|
supplierAwarded: toText(record.agency),
|
|
documents: sanitizeDocuments(record),
|
|
rawSourceUrl: requestUrl.toString(),
|
|
rawPayload: {
|
|
source: "licitaya-backfill",
|
|
licitayaState: toText(record.state),
|
|
licitayaCity: toText(record.city),
|
|
record,
|
|
},
|
|
lastSeenAt: new Date(),
|
|
},
|
|
});
|
|
|
|
if (existing) {
|
|
summary.updated += 1;
|
|
} else {
|
|
summary.inserted += 1;
|
|
}
|
|
}
|
|
|
|
const totalPages = Number.parseInt(toText(payload.total_pages) || "1", 10);
|
|
|
|
if (!Number.isFinite(totalPages) || page >= totalPages || page >= args.maxPages) {
|
|
break;
|
|
}
|
|
|
|
await delay(args.delayMs);
|
|
}
|
|
|
|
if (!dateHadResults) {
|
|
summary.emptyDates += 1;
|
|
}
|
|
}
|
|
|
|
console.log(JSON.stringify(summary, null, 2));
|
|
} finally {
|
|
await prisma.$disconnect();
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
});
|