ray/aliyunApsSkill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增量

Browse Source
This commit is contained in:
ray
2026-04-21 21:16:56 +08:00
parent aa67b0e37e
commit 19e8a833ba
9 changed files with 777 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
aliyun-sync/aliyun-aps-sync/src/sync.js
View File

@@ -1,6 +1,8 @@
import { chromium } from 'playwright';
import cron from 'node-cron';
import fs from 'node:fs';
import path from 'node:path';
import readline from 'node:readline';
import { execSync } from 'node:child_process';
import { config, datasets } from './config.js';
import { sendLoginAlert } from './notify.js';
@@ -17,6 +19,106 @@ import {
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let _context = null;
let _runtimeController = null;
const AUTH_PAGE_KEYWORDS = [
'RAM 用户登录',
'主账号登录',
'钉钉扫码登录',
'用户名',
'下一步',
'登录并使用 RAM',
];
async function closeContextIfNeeded() {
if (!_context) return;
await _context.close();
_context = null;
}
function getRuntimeController() {
if (_runtimeController) return _runtimeController;
let paused = false;
let terminated = false;
let keypressBound = false;
const onKeypress = (_str, key = {}) => {
if (key.name === 'f7') {
if (!paused) {
paused = true;
console.log('[控制] 已暂停（F7）。按 F8 继续，按 F9 终止。');
}
return;
}
if (key.name === 'f8') {
if (paused) {
paused = false;
console.log('[控制] 已继续（F8）。');
}
return;
}
if (key.name === 'f9') {
terminated = true;
paused = false;
console.log('[控制] 已请求终止（F9），将在安全检查点停止。');
}
};
const bind = () => {
if (keypressBound || !process.stdin.isTTY) return;
readline.emitKeypressEvents(process.stdin);
if (typeof process.stdin.setRawMode === 'function') {
process.stdin.setRawMode(true);
}
process.stdin.resume();
process.stdin.on('keypress', onKeypress);
keypressBound = true;
console.log('[控制] 热键已启用：F7 暂停 / F8 继续 / F9 终止');
};
const unbind = () => {
if (!keypressBound) return;
process.stdin.off('keypress', onKeypress);
if (process.stdin.isTTY && typeof process.stdin.setRawMode === 'function') {
process.stdin.setRawMode(false);
}
keypressBound = false;
};
const waitIfPaused = async (label = '任务') => {
if (terminated) {
throw new Error(`[控制] 已终止：${label}`);
}
while (paused) {
await sleep(300);
if (terminated) {
throw new Error(`[控制] 已终止：${label}`);
}
}
};
const throwIfTerminated = (label = '任务') => {
if (terminated) {
throw new Error(`[控制] 已终止：${label}`);
}
};
_runtimeController = {
bind,
unbind,
waitIfPaused,
throwIfTerminated,
};
return _runtimeController;
}
async function runtimeCheckpoint(label) {
const controller = getRuntimeController();
controller.throwIfTerminated(label);
await controller.waitIfPaused(label);
}
async function getContext() {
if (_context) return _context;
@@ -26,24 +128,103 @@ async function getContext() {
acceptDownloads: true,
downloadsPath: config.downloadDir,
});
await restoreStorageState(_context);
return _context;
}
async function restoreStorageState(context) {
if (!fs.existsSync(config.storageStateFile)) {
return;
}
try {
const state = JSON.parse(fs.readFileSync(config.storageStateFile, 'utf-8'));
if (Array.isArray(state.cookies) && state.cookies.length > 0) {
await context.addCookies(state.cookies);
console.log(`[storageState] 已恢复 ${state.cookies.length} 个 cookie`);
}
} catch (error) {
console.warn(`[storageState] 恢复失败，继续使用 .browser profile: ${error.message}`);
}
}
async function saveStorageState(context) {
await context.storageState({ path: config.storageStateFile });
console.log(`[storageState] 已保存登录态快照: ${config.storageStateFile}`);
}
async function getPageBodyPreview(page) {
return page
.evaluate(() => document.body?.innerText?.substring(0, 500) || '(空)')
.catch(() => '(无法获取)');
}
function isAuthUrl(url) {
return /account\.aliyun\.com|signin\.aliyun\.com/.test(url)
|| url.includes('login.htm')
|| url.includes('/#/signin');
}
function hasAuthKeywords(text) {
return AUTH_PAGE_KEYWORDS.some((keyword) => text.includes(keyword));
}
async function detectAuthRedirect(page) {
const currentUrl = page.url();
const bodyText = await getPageBodyPreview(page);
return {
currentUrl,
bodyText,
isAuthPage: isAuthUrl(currentUrl) || hasAuthKeywords(bodyText),
};
}
async function ensureDatasetAccessible(page, dataset, timeout = 120000, options = {}) {
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
await waitUntilReady(page, dataset.heading, timeout, options);
}
export async function login() {
const runtimeController = getRuntimeController();
runtimeController.bind();
const context = await getContext();
const cleanupAndExit = async (signal) => {
console.log(`[login] 收到 ${signal}，正在保存登录态并关闭浏览器...`);
await closeContextIfNeeded();
process.exit(130);
};
const page = context.pages()[0] || (await context.newPage());
await page.goto(datasets.customers.url, { waitUntil: 'domcontentloaded' });
console.log('请在打开的浏览器里完成阿里云伙伴中心登录，然后回到终端按 Ctrl+C 结束。');
await waitUntilReady(page, datasets.customers.heading, 10 * 60 * 1000);
console.log('登录态已写入 .browser 目录，后续可直接执行 npm run sync。');
const onSigint = () => {
void cleanupAndExit('SIGINT');
};
const onSigterm = () => {
void cleanupAndExit('SIGTERM');
};
// 必须正常关闭 context，否则登录态不会持久化到磁盘
await context.close();
_context = null;
process.once('SIGINT', onSigint);
process.once('SIGTERM', onSigterm);
try {
const page = context.pages()[0] || (await context.newPage());
await page.goto(datasets.customers.url, { waitUntil: 'domcontentloaded' });
console.log('请在打开的浏览器里完成阿里云伙伴中心登录。检测到进入“我的客户”和“账单查询”页面后，脚本会自动保存登录态并关闭浏览器。');
await waitUntilReady(page, datasets.customers.heading, 10 * 60 * 1000, { allowInteractiveAuth: true });
console.log('[login] 我的客户页验证通过，继续验证账单页登录态...');
await ensureDatasetAccessible(page, datasets.bills, 60 * 1000, { allowInteractiveAuth: true });
await sleep(1000);
await saveStorageState(context);
console.log('登录态已写入 .browser 目录，且已验证“我的客户”和“账单查询”页面可访问，后续可直接执行 npm run sync 或 npm run bills。');
} finally {
process.off('SIGINT', onSigint);
process.off('SIGTERM', onSigterm);
await closeContextIfNeeded();
runtimeController.unbind();
}
}
export async function syncAll() {
const runtimeController = getRuntimeController();
runtimeController.bind();
const context = await getContext();
try {
@@ -67,11 +248,36 @@ export async function syncAll() {
return summary;
} finally {
if (config.closeBrowser) {
await context.close();
_context = null;
await closeContextIfNeeded();
} else {
console.log('浏览器保持运行');
}
runtimeController.unbind();
}
}
export async function syncBillsOnly() {
const runtimeController = getRuntimeController();
runtimeController.bind();
const context = await getContext();
try {
const summary = { startedAt: new Date().toISOString(), datasets: {} };
const page = context.pages()[0] || (await context.newPage());
summary.datasets.bills = await syncBills(page);
summary.finishedAt = new Date().toISOString();
const stamp = nowStamp();
saveRunSummary(stamp, summary);
return summary;
} finally {
if (config.closeBrowser) {
await closeContextIfNeeded();
} else {
console.log('浏览器保持运行');
}
runtimeController.unbind();
}
}
@@ -106,6 +312,7 @@ export async function scheduleSync() {
}
async function syncCustomers(page) {
await runtimeCheckpoint('同步客户');
const dataset = datasets.customers;
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
await waitUntilReady(page, dataset.heading);
@@ -115,6 +322,7 @@ async function syncCustomers(page) {
}
async function syncCustomerDetails(page) {
await runtimeCheckpoint('同步客户详情');
const dataset = datasets.customerDetails;
const customersState = loadCurrentState('customers');
const allAccountIds = collectValidAccountIds(customersState.records || []);
@@ -130,6 +338,7 @@ async function syncCustomerDetails(page) {
'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/my_customer/~/customer/';
for (let index = 0; index < allAccountIds.length; index += 1) {
await runtimeCheckpoint(`客户详情 ${index + 1}/${allAccountIds.length}`);
const accountId = allAccountIds[index];
console.log(`[客户详情] ${index + 1}/${allAccountIds.length} accountId=${accountId}`);
@@ -158,23 +367,20 @@ async function syncCustomerDetails(page) {
}
async function syncOrders(page) {
await runtimeCheckpoint('同步订单');
const dataset = datasets.orders;
let windows;
if (config.fullSync) {
windows = buildMonthlyDateWindows(config.orderStartDate);
} else {
// 增量模式：只查前一天
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
const dateStr = formatDate(yesterday);
windows = [{ windowStart: dateStr, windowEnd: dateStr, start: dateStr, end: dateStr }];
console.log(`[增量模式] 订单仅查询: ${dateStr}`);
windows = buildIncrementalOrderWindows();
}
const allRecords = [];
for (const window of windows) {
await runtimeCheckpoint(`订单窗口 ${window.start} ~ ${window.end}`);
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
await waitUntilReady(page, dataset.heading);
await setDateRange(page, window.start, window.end);
@@ -187,7 +393,40 @@ async function syncOrders(page) {
return persistDataset(dataset, dedupeByHash(allRecords), {});
}
function buildIncrementalOrderWindows() {
const configuredStartDate = normalizeConfiguredDate(config.incrementalOrderStartDate);
if (configuredStartDate) {
const windows = buildMonthlyDateWindows(configuredStartDate);
console.log(`[增量模式] 订单从指定日期开始查询: ${configuredStartDate}`);
return windows;
}
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
const dateStr = formatDate(yesterday);
console.log(`[增量模式] 订单仅查询: ${dateStr}`);
return [{ windowStart: dateStr, windowEnd: dateStr, start: dateStr, end: dateStr }];
}
function normalizeConfiguredDate(value) {
const normalized = String(value || '').trim();
if (!normalized) {
return '';
}
if (!/^\d{4}-\d{2}-\d{2}$/.test(normalized)) {
throw new Error(`ALIYUN_APS_INCREMENTAL_ORDER_START_DATE 格式无效: ${normalized}，期望 YYYY-MM-DD`);
}
const parsed = new Date(`${normalized}T00:00:00+08:00`);
if (Number.isNaN(parsed.getTime())) {
throw new Error(`ALIYUN_APS_INCREMENTAL_ORDER_START_DATE 不是有效日期: ${normalized}`);
}
return normalized;
}
async function syncBills(page) {
await runtimeCheckpoint('同步账单');
const dataset = datasets.bills;
let months;
let latestConsumptionDate = null;
@@ -205,6 +444,7 @@ async function syncBills(page) {
const allRecords = [];
for (const month of months) {
await runtimeCheckpoint(`账单月份 ${month}`);
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
await waitUntilReady(page, dataset.heading);
await setMonthValue(page, month);
@@ -247,6 +487,7 @@ function isAfterLatestConsumptionDate(record, latestConsumptionDate) {
}
async function syncOrderDetails(page, cachedOrderIds) {
await runtimeCheckpoint('同步订单详情');
const dataset = datasets.orderDetails;
// 使用传入的 orderId 列表（在 syncOrders 覆盖 orders.json 之前缓存的）
@@ -262,6 +503,7 @@ async function syncOrderDetails(page, cachedOrderIds) {
const detailBaseUrl = 'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/order/~/costCenter/order/detail/';
for (let index = 0; index < allOrderIds.length; index += 1) {
await runtimeCheckpoint(`订单详情 ${index + 1}/${allOrderIds.length}`);
const orderId = allOrderIds[index];
console.log(`[订单详情] ${index + 1}/${allOrderIds.length} orderId=${orderId}`);
@@ -304,11 +546,30 @@ function persistDataset(dataset, records, context) {
};
}
async function waitUntilReady(page, heading, timeout = 120000) {
async function waitUntilReady(page, heading, timeout = 120000, options = {}) {
await runtimeCheckpoint(`等待页面 ${heading}`);
const { allowInteractiveAuth = false } = options;
await page.waitForLoadState('domcontentloaded');
console.log(`[waitUntilReady] 当前URL: ${page.url()}`);
console.log(`[waitUntilReady] 等待页面出现: "${heading}"`);
const initialState = await detectAuthRedirect(page);
if (initialState.isAuthPage) {
console.error(`[waitUntilReady] 检测到登录页/鉴权页: ${initialState.currentUrl}`);
console.error(`[waitUntilReady] 页面内容前500字: ${initialState.bodyText}`);
if (!allowInteractiveAuth && isAuthUrl(initialState.currentUrl)) {
try {
await sendLoginAlert(initialState.currentUrl);
} catch (notifyErr) {
console.error('[通知] 发送登录提醒失败:', notifyErr.message);
}
}
if (!allowInteractiveAuth) {
throw new Error(`当前页面仍处于登录/鉴权页，无法进入「${heading}」。请重新执行 npm run login，并确认该账号对该页面有访问权限。`);
}
console.log(`[waitUntilReady] 允许交互式登录，等待用户完成认证后进入「${heading}」...`);
}
try {
await page.waitForFunction(
(text) => document.body && document.body.innerText.includes(text),
@@ -317,22 +578,26 @@ async function waitUntilReady(page, heading, timeout = 120000) {
);
} catch (err) {
// 超时时打印诊断信息
const currentUrl = page.url();
const bodyText = await page.evaluate(() => document.body?.innerText?.substring(0, 500) || '(空)').catch(() => '(无法获取)');
const { currentUrl, bodyText, isAuthPage } = await detectAuthRedirect(page);
console.error(`[waitUntilReady] 超时！当前URL: ${currentUrl}`);
console.error(`[waitUntilReady] 页面内容前500字: ${bodyText}`);
if (currentUrl.includes('signin')) {
if (isAuthPage && !allowInteractiveAuth) {
try {
await sendLoginAlert();
await sendLoginAlert(currentUrl);
} catch (notifyErr) {
console.error('[通知] 发送登录提醒失败:', notifyErr.message);
}
throw new Error(`当前页面停留在登录/鉴权页，未能进入「${heading}」。请重新执行 npm run login，并确认该账号对该页面有访问权限。`);
}
if (isAuthPage && allowInteractiveAuth) {
throw new Error(`交互式登录超时，仍未进入「${heading}」。请确认已在浏览器中完成 RAM/阿里云登录，并且当前账号有访问该页面的权限。`);
}
throw err;
}
if ((await page.locator('text=登录').count()) > 0 && page.url().includes('login')) {
throw new Error('当前未登录，请先执行 npm run login');
const finalState = await detectAuthRedirect(page);
if (finalState.isAuthPage && !allowInteractiveAuth) {
throw new Error(`当前页面仍处于登录/鉴权页，未成功进入「${heading}」。请重新执行 npm run login，并确认该账号对该页面有访问权限。`);
}
await sleep(1500);
}
@@ -342,6 +607,7 @@ async function scrapePagedTable(page, dataset, context) {
const visited = new Set();
while (true) {
await runtimeCheckpoint(`抓取 ${dataset.name} 分页`);
await waitForTableRows(page);
const pageData = await extractTable(page);
const pageNum = await currentPageNumber(page);
@@ -403,6 +669,7 @@ async function extractTable(page) {
}
async function waitForTableRows(page) {
await runtimeCheckpoint('等待表格加载');
await page.waitForFunction(() => document.querySelectorAll('table tbody tr').length > 0, null, { timeout: 120000 });
await sleep(800);
}
@@ -414,6 +681,7 @@ async function currentPageNumber(page) {
}
async function gotoNextPage(page) {
await runtimeCheckpoint('翻页');
const before = await currentPageNumber(page);
// 用 Playwright locator 定位"下一页"按钮
@@ -439,6 +707,7 @@ async function gotoNextPage(page) {
}
async function trySetPageSize(page, pageSize) {
await runtimeCheckpoint(`设置每页 ${pageSize}`);
const input = page.locator('input[aria-label="请选择每页显示几条"]').first();
if ((await input.count()) === 0) return;
await input.click().catch(() => null);
@@ -453,6 +722,7 @@ async function trySetPageSize(page, pageSize) {
}
async function setDateRange(page, start, end) {
await runtimeCheckpoint(`设置订单日期 ${start} ~ ${end}`);
console.log(`[订单日期] 设置: ${start} ~ ${end}`);
await _fillDateRange(page, start, end);
@@ -474,6 +744,7 @@ async function setDateRange(page, start, end) {
}
async function _fillDateRange(page, start, end, startFirst = false) {
await runtimeCheckpoint('填写订单日期');
const trigger = page.locator('input[placeholder="结束日期"]');
await trigger.click();
await sleep(1000);
@@ -520,6 +791,7 @@ async function _fillDateRange(page, start, end, startFirst = false) {
}
async function setMonthValue(page, month) {
await runtimeCheckpoint(`设置账单月份 ${month}`);
// 先尝试按 inputValue 匹配 YYYY-MM 格式
const inputs = page.locator('input');
const total = await inputs.count();
@@ -568,6 +840,7 @@ async function setMonthValue(page, month) {
* 即使面板弹出，快速键入 + Tab 也能在面板滚动前完成提交并关闭。
*/
async function typeIntoDateInput(locator, value, page) {
await runtimeCheckpoint(`填写日期输入 ${value}`);
// 移除 readonly
await locator.evaluate((node) => node.removeAttribute('readonly'));
@@ -599,6 +872,7 @@ async function typeIntoDateInput(locator, value, page) {
}
async function clickQuery(page) {
await runtimeCheckpoint('点击查询');
const button = page.locator('button:has-text("查询")').first();
await button.click();
await sleep(1800);