订单每页20,和客户详情单独提交
This commit is contained in:
@@ -74,6 +74,7 @@ export const datasets = {
|
|||||||
return {
|
return {
|
||||||
loginName: loginName.replace(/\s+/g, ''),
|
loginName: loginName.replace(/\s+/g, ''),
|
||||||
accountId,
|
accountId,
|
||||||
|
listPageNum: context.pageNum || '',
|
||||||
realName: record['UID实名认证名称'] || '',
|
realName: record['UID实名认证名称'] || '',
|
||||||
reportSource: record['报备来源'] || '',
|
reportSource: record['报备来源'] || '',
|
||||||
reportType: record['报备类型'] || '',
|
reportType: record['报备类型'] || '',
|
||||||
@@ -101,7 +102,7 @@ export const datasets = {
|
|||||||
name: 'orders',
|
name: 'orders',
|
||||||
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
||||||
heading: '订单查询',
|
heading: '订单查询',
|
||||||
pageSize: 100,
|
pageSize: 20,
|
||||||
uniqueKey: (record) => record.orderId || record.__hash,
|
uniqueKey: (record) => record.orderId || record.__hash,
|
||||||
normalize: (record, context) => ({
|
normalize: (record, context) => ({
|
||||||
orderId: record['订单号'] || '',
|
orderId: record['订单号'] || '',
|
||||||
|
|||||||
@@ -672,30 +672,44 @@ async function syncCustomerDetails(page) {
|
|||||||
await runtimeCheckpoint('同步客户详情');
|
await runtimeCheckpoint('同步客户详情');
|
||||||
const dataset = datasets.customerDetails;
|
const dataset = datasets.customerDetails;
|
||||||
const customersState = loadCurrentState('customers', datasets.customers.uniqueKey);
|
const customersState = loadCurrentState('customers', datasets.customers.uniqueKey);
|
||||||
const allAccountIds = collectValidAccountIds(customersState.records || []);
|
const customerTargets = collectCustomerDetailTargets(customersState.records || []);
|
||||||
|
|
||||||
if (allAccountIds.length === 0) {
|
if (customerTargets.length === 0) {
|
||||||
console.log('[客户详情] 本地无有效客户 accountId,跳过');
|
console.log('[客户详情] 本地无有效客户定位信息,跳过');
|
||||||
return persistDataset(dataset, [], {});
|
return persistDataset(dataset, [], {});
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`[客户详情] 共 ${allAccountIds.length} 个客户需要获取详情`);
|
console.log(`[客户详情] 共 ${customerTargets.length} 个客户需要获取详情`);
|
||||||
const allDetails = [];
|
const allDetails = [];
|
||||||
const detailBaseUrl =
|
let currentListPage = 0;
|
||||||
'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/my_customer/~/customer/';
|
|
||||||
|
|
||||||
for (let index = 0; index < allAccountIds.length; index += 1) {
|
await page.goto(datasets.customers.url, { waitUntil: 'domcontentloaded' });
|
||||||
await runtimeCheckpoint(`客户详情 ${index + 1}/${allAccountIds.length}`);
|
await waitUntilReady(page, datasets.customers.heading);
|
||||||
const accountId = allAccountIds[index];
|
await trySetPageSize(page, datasets.customers.pageSize);
|
||||||
console.log(`[客户详情] ${index + 1}/${allAccountIds.length} accountId=${accountId}`);
|
|
||||||
|
for (let index = 0; index < customerTargets.length; index += 1) {
|
||||||
|
await runtimeCheckpoint(`客户详情 ${index + 1}/${customerTargets.length}`);
|
||||||
|
const target = customerTargets[index];
|
||||||
|
console.log(`[客户详情] ${index + 1}/${customerTargets.length} accountId=${target.accountId} page=${target.pageNum}`);
|
||||||
const pauseMs = randomIntBetween(1000, 3000);
|
const pauseMs = randomIntBetween(1000, 3000);
|
||||||
console.log(`[客户详情] 随机等待 ${pauseMs}ms 后继续`);
|
console.log(`[客户详情] 随机等待 ${pauseMs}ms 后继续`);
|
||||||
await sleep(pauseMs);
|
await sleep(pauseMs);
|
||||||
|
|
||||||
// 先跳 about:blank 再跳详情URL(强制 SPA 完整重新加载)
|
if (target.pageNum > 0 && currentListPage !== target.pageNum) {
|
||||||
await page.goto('about:blank');
|
const reached = await jumpToPage(page, target.pageNum);
|
||||||
await sleep(300);
|
if (!reached) {
|
||||||
await page.goto(`${detailBaseUrl}${accountId}`, { waitUntil: 'domcontentloaded' });
|
console.warn(`[客户详情] 无法跳到第 ${target.pageNum} 页,跳过 ${target.accountId}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
currentListPage = target.pageNum;
|
||||||
|
await waitForTableRows(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
const clicked = await clickCustomerDetailFromList(page, target);
|
||||||
|
if (!clicked) {
|
||||||
|
console.warn(`[客户详情] 列表中未找到 accountId=${target.accountId},跳过`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await page.waitForFunction(
|
await page.waitForFunction(
|
||||||
@@ -705,16 +719,23 @@ async function syncCustomerDetails(page) {
|
|||||||
);
|
);
|
||||||
await sleep(1000);
|
await sleep(1000);
|
||||||
} catch {
|
} catch {
|
||||||
console.warn(`[客户详情] ${accountId} 详情页加载超时,跳过`);
|
console.warn(`[客户详情] ${target.accountId} 详情页加载超时,跳过`);
|
||||||
|
await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => null);
|
||||||
|
await waitUntilReady(page, datasets.customers.heading).catch(() => null);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const detail = await extractCustomerDetail(page);
|
const detail = await extractCustomerDetail(page);
|
||||||
allDetails.push({ ...detail, __context: { accountId } });
|
allDetails.push({ ...detail, __context: { accountId: target.accountId } });
|
||||||
if (hasDbConfig()) {
|
if (hasDbConfig()) {
|
||||||
const normalizedDetail = normalizeDatasetRecords(dataset, [{ ...detail, __context: { accountId } }], {});
|
const normalizedDetail = normalizeDatasetRecords(dataset, [{ ...detail, __context: { accountId: target.accountId } }], {});
|
||||||
await upsertCustomerDetails(normalizedDetail);
|
await upsertCustomerDetails(normalizedDetail);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => null);
|
||||||
|
await waitUntilReady(page, datasets.customers.heading).catch(() => null);
|
||||||
|
await trySetPageSize(page, datasets.customers.pageSize).catch(() => null);
|
||||||
|
currentListPage = target.pageNum;
|
||||||
}
|
}
|
||||||
|
|
||||||
return persistDataset(dataset, dedupeByHash(allDetails), {});
|
return persistDataset(dataset, dedupeByHash(allDetails), {});
|
||||||
@@ -1219,7 +1240,7 @@ async function scrapePagedTable(page, dataset, context, options = {}) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
visited.add(pageKey);
|
visited.add(pageKey);
|
||||||
const pageRows = pageData.rows.map((row) => ({ ...row, __context: context }));
|
const pageRows = pageData.rows.map((row) => ({ ...row, __context: { ...context, pageNum } }));
|
||||||
pages.push(...pageRows);
|
pages.push(...pageRows);
|
||||||
if (onPage) {
|
if (onPage) {
|
||||||
await onPage({ pageData, pageNum, pageRows });
|
await onPage({ pageData, pageNum, pageRows });
|
||||||
@@ -1668,6 +1689,54 @@ function collectValidAccountIds(records) {
|
|||||||
return ids;
|
return ids;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function collectCustomerDetailTargets(records) {
|
||||||
|
const targets = [];
|
||||||
|
const seen = new Set();
|
||||||
|
for (const record of records) {
|
||||||
|
const accountId = String(record.accountId || '').trim();
|
||||||
|
const loginName = String(record.loginName || '').trim();
|
||||||
|
const pageNum = Number.parseInt(String(record.listPageNum || 0), 10) || 0;
|
||||||
|
if (!accountId || !isValidAccountId(accountId) || pageNum <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (seen.has(accountId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(accountId);
|
||||||
|
targets.push({ accountId, loginName, pageNum });
|
||||||
|
}
|
||||||
|
return targets.sort((a, b) => a.pageNum - b.pageNum);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clickCustomerDetailFromList(page, target) {
|
||||||
|
const clicked = await page.evaluate(({ accountId, loginName }) => {
|
||||||
|
const normalize = (value) => String(value || '').replace(/\s+/g, '').trim();
|
||||||
|
const rows = Array.from(document.querySelectorAll('table tbody tr'));
|
||||||
|
const targetRow = rows.find((row) => {
|
||||||
|
const text = normalize(row.innerText || row.textContent || '');
|
||||||
|
return text.includes(accountId) || (loginName && text.includes(loginName));
|
||||||
|
});
|
||||||
|
if (!targetRow) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const detailButton = Array.from(targetRow.querySelectorAll('button, a, span'))
|
||||||
|
.find((node) => /详情/.test(String(node.textContent || '').trim()));
|
||||||
|
if (!detailButton) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
detailButton.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
|
||||||
|
detailButton.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||||
|
return true;
|
||||||
|
}, target).catch(() => false);
|
||||||
|
|
||||||
|
if (clicked) {
|
||||||
|
await sleep(1200);
|
||||||
|
}
|
||||||
|
return clicked;
|
||||||
|
}
|
||||||
|
|
||||||
function isValidOrderId(orderId) {
|
function isValidOrderId(orderId) {
|
||||||
const value = String(orderId || '').trim();
|
const value = String(orderId || '').trim();
|
||||||
if (!value) return false;
|
if (!value) return false;
|
||||||
|
|||||||
Reference in New Issue
Block a user