订单每页20,和客户详情单独提交

This commit is contained in:
ray
2026-05-06 15:38:24 +08:00
parent d14ed90597
commit 1c92478867
2 changed files with 89 additions and 19 deletions

View File

@@ -74,6 +74,7 @@ export const datasets = {
return { return {
loginName: loginName.replace(/\s+/g, ''), loginName: loginName.replace(/\s+/g, ''),
accountId, accountId,
listPageNum: context.pageNum || '',
realName: record['UID实名认证名称'] || '', realName: record['UID实名认证名称'] || '',
reportSource: record['报备来源'] || '', reportSource: record['报备来源'] || '',
reportType: record['报备类型'] || '', reportType: record['报备类型'] || '',
@@ -101,7 +102,7 @@ export const datasets = {
name: 'orders', name: 'orders',
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`, url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
heading: '订单查询', heading: '订单查询',
pageSize: 100, pageSize: 20,
uniqueKey: (record) => record.orderId || record.__hash, uniqueKey: (record) => record.orderId || record.__hash,
normalize: (record, context) => ({ normalize: (record, context) => ({
orderId: record['订单号'] || '', orderId: record['订单号'] || '',

View File

@@ -672,30 +672,44 @@ async function syncCustomerDetails(page) {
await runtimeCheckpoint('同步客户详情'); await runtimeCheckpoint('同步客户详情');
const dataset = datasets.customerDetails; const dataset = datasets.customerDetails;
const customersState = loadCurrentState('customers', datasets.customers.uniqueKey); const customersState = loadCurrentState('customers', datasets.customers.uniqueKey);
const allAccountIds = collectValidAccountIds(customersState.records || []); const customerTargets = collectCustomerDetailTargets(customersState.records || []);
if (allAccountIds.length === 0) { if (customerTargets.length === 0) {
console.log('[客户详情] 本地无有效客户 accountId,跳过'); console.log('[客户详情] 本地无有效客户定位信息,跳过');
return persistDataset(dataset, [], {}); return persistDataset(dataset, [], {});
} }
console.log(`[客户详情] 共 ${allAccountIds.length} 个客户需要获取详情`); console.log(`[客户详情] 共 ${customerTargets.length} 个客户需要获取详情`);
const allDetails = []; const allDetails = [];
const detailBaseUrl = let currentListPage = 0;
'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/my_customer/~/customer/';
for (let index = 0; index < allAccountIds.length; index += 1) { await page.goto(datasets.customers.url, { waitUntil: 'domcontentloaded' });
await runtimeCheckpoint(`客户详情 ${index + 1}/${allAccountIds.length}`); await waitUntilReady(page, datasets.customers.heading);
const accountId = allAccountIds[index]; await trySetPageSize(page, datasets.customers.pageSize);
console.log(`[客户详情] ${index + 1}/${allAccountIds.length} accountId=${accountId}`);
for (let index = 0; index < customerTargets.length; index += 1) {
await runtimeCheckpoint(`客户详情 ${index + 1}/${customerTargets.length}`);
const target = customerTargets[index];
console.log(`[客户详情] ${index + 1}/${customerTargets.length} accountId=${target.accountId} page=${target.pageNum}`);
const pauseMs = randomIntBetween(1000, 3000); const pauseMs = randomIntBetween(1000, 3000);
console.log(`[客户详情] 随机等待 ${pauseMs}ms 后继续`); console.log(`[客户详情] 随机等待 ${pauseMs}ms 后继续`);
await sleep(pauseMs); await sleep(pauseMs);
// 先跳 about:blank 再跳详情URL强制 SPA 完整重新加载) if (target.pageNum > 0 && currentListPage !== target.pageNum) {
await page.goto('about:blank'); const reached = await jumpToPage(page, target.pageNum);
await sleep(300); if (!reached) {
await page.goto(`${detailBaseUrl}${accountId}`, { waitUntil: 'domcontentloaded' }); console.warn(`[客户详情] 无法跳到第 ${target.pageNum} 页,跳过 ${target.accountId}`);
continue;
}
currentListPage = target.pageNum;
await waitForTableRows(page);
}
const clicked = await clickCustomerDetailFromList(page, target);
if (!clicked) {
console.warn(`[客户详情] 列表中未找到 accountId=${target.accountId},跳过`);
continue;
}
try { try {
await page.waitForFunction( await page.waitForFunction(
@@ -705,16 +719,23 @@ async function syncCustomerDetails(page) {
); );
await sleep(1000); await sleep(1000);
} catch { } catch {
console.warn(`[客户详情] ${accountId} 详情页加载超时,跳过`); console.warn(`[客户详情] ${target.accountId} 详情页加载超时,跳过`);
await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => null);
await waitUntilReady(page, datasets.customers.heading).catch(() => null);
continue; continue;
} }
const detail = await extractCustomerDetail(page); const detail = await extractCustomerDetail(page);
allDetails.push({ ...detail, __context: { accountId } }); allDetails.push({ ...detail, __context: { accountId: target.accountId } });
if (hasDbConfig()) { if (hasDbConfig()) {
const normalizedDetail = normalizeDatasetRecords(dataset, [{ ...detail, __context: { accountId } }], {}); const normalizedDetail = normalizeDatasetRecords(dataset, [{ ...detail, __context: { accountId: target.accountId } }], {});
await upsertCustomerDetails(normalizedDetail); await upsertCustomerDetails(normalizedDetail);
} }
await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => null);
await waitUntilReady(page, datasets.customers.heading).catch(() => null);
await trySetPageSize(page, datasets.customers.pageSize).catch(() => null);
currentListPage = target.pageNum;
} }
return persistDataset(dataset, dedupeByHash(allDetails), {}); return persistDataset(dataset, dedupeByHash(allDetails), {});
@@ -1219,7 +1240,7 @@ async function scrapePagedTable(page, dataset, context, options = {}) {
break; break;
} }
visited.add(pageKey); visited.add(pageKey);
const pageRows = pageData.rows.map((row) => ({ ...row, __context: context })); const pageRows = pageData.rows.map((row) => ({ ...row, __context: { ...context, pageNum } }));
pages.push(...pageRows); pages.push(...pageRows);
if (onPage) { if (onPage) {
await onPage({ pageData, pageNum, pageRows }); await onPage({ pageData, pageNum, pageRows });
@@ -1668,6 +1689,54 @@ function collectValidAccountIds(records) {
return ids; return ids;
} }
function collectCustomerDetailTargets(records) {
const targets = [];
const seen = new Set();
for (const record of records) {
const accountId = String(record.accountId || '').trim();
const loginName = String(record.loginName || '').trim();
const pageNum = Number.parseInt(String(record.listPageNum || 0), 10) || 0;
if (!accountId || !isValidAccountId(accountId) || pageNum <= 0) {
continue;
}
if (seen.has(accountId)) {
continue;
}
seen.add(accountId);
targets.push({ accountId, loginName, pageNum });
}
return targets.sort((a, b) => a.pageNum - b.pageNum);
}
async function clickCustomerDetailFromList(page, target) {
const clicked = await page.evaluate(({ accountId, loginName }) => {
const normalize = (value) => String(value || '').replace(/\s+/g, '').trim();
const rows = Array.from(document.querySelectorAll('table tbody tr'));
const targetRow = rows.find((row) => {
const text = normalize(row.innerText || row.textContent || '');
return text.includes(accountId) || (loginName && text.includes(loginName));
});
if (!targetRow) {
return false;
}
const detailButton = Array.from(targetRow.querySelectorAll('button, a, span'))
.find((node) => /详情/.test(String(node.textContent || '').trim()));
if (!detailButton) {
return false;
}
detailButton.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
detailButton.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
return true;
}, target).catch(() => false);
if (clicked) {
await sleep(1200);
}
return clicked;
}
function isValidOrderId(orderId) { function isValidOrderId(orderId) {
const value = String(orderId || '').trim(); const value = String(orderId || '').trim();
if (!value) return false; if (!value) return false;