ray/aliyunApsSkill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增量

Browse Source
This commit is contained in:
ray
2026-04-21 21:16:56 +08:00
parent aa67b0e37e
commit 19e8a833ba
9 changed files with 777 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
aliyun-sync/aps-aliyun-sync/aps_db_sync.py
View File

@@ -35,6 +35,7 @@ from pymysql.cursors import DictCursor
JsonDict = dict[str, object]
JsonList = list[JsonDict]
StatsDict = dict[str, int]
SyncTarget = str
class DbConfig(TypedDict):
@@ -48,7 +49,7 @@ class DbConfig(TypedDict):
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
DB_CONFIG = {
DB_CONFIG: DbConfig = {
"host": "172.27.137.236",
"port": 3306,
"user": "ray",
@@ -62,6 +63,19 @@ LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger("aps_sync")
SYNC_TARGET_ALL = "all"
SYNC_TARGET_CUSTOMER = "customer"
SYNC_TARGET_ORDER = "order"
SYNC_TARGET_ORDERDETAILS = "orderdetails"
SYNC_TARGET_BILLS = "bills"
VALID_SYNC_TARGETS = {
SYNC_TARGET_ALL,
SYNC_TARGET_CUSTOMER,
SYNC_TARGET_ORDER,
SYNC_TARGET_ORDERDETAILS,
SYNC_TARGET_BILLS,
}
# ---------------------------------------------------------------------------
# Schema DDL
# ---------------------------------------------------------------------------
@@ -382,6 +396,18 @@ def is_valid_order_id(order_id: str | None) -> bool:
return bool(order_id and order_id.isdigit())
def normalize_sync_target(sync_target: str | None) -> SyncTarget:
if sync_target is None:
return SYNC_TARGET_ALL
normalized = sync_target.strip().lower()
if not normalized:
return SYNC_TARGET_ALL
if normalized not in VALID_SYNC_TARGETS:
valid_targets = ", ".join(sorted(VALID_SYNC_TARGETS))
raise ValueError(f"Invalid sync target: {sync_target}. Expected one of: {valid_targets}")
return normalized
# ---------------------------------------------------------------------------
# Sync logic
# ---------------------------------------------------------------------------
@@ -465,7 +491,7 @@ class APSSyncer:
return [cast(JsonDict, record) for record in data_list if isinstance(record, dict)]
return []
def resolve_data_files(self, data_dir: str) -> tuple[Path, Path, Path, Path, Path]:
def resolve_data_files(self, data_dir: str, sync_target: SyncTarget = SYNC_TARGET_ALL) -> tuple[Path, Path, Path, Path, Path]:
root = Path(data_dir)
if not root.exists() or not root.is_dir():
raise FileNotFoundError(f"Data directory not found: {root}")
@@ -475,11 +501,33 @@ class APSSyncer:
order_details_file = root / "orderDetails.json"
bills_file = root / "bills.json"
customer_details_file = root / "customerDetails.json"
for fp in (customers_file, orders_file, order_details_file, bills_file):
required_files_by_target = {
SYNC_TARGET_ALL: (customers_file, orders_file, order_details_file, bills_file),
SYNC_TARGET_CUSTOMER: (customers_file,),
SYNC_TARGET_ORDER: (orders_file,),
SYNC_TARGET_ORDERDETAILS: (order_details_file,),
SYNC_TARGET_BILLS: (bills_file,),
}
for fp in required_files_by_target[sync_target]:
if not fp.exists():
raise FileNotFoundError(f"Required JSON file not found: {fp}")
return customers_file, orders_file, order_details_file, bills_file, customer_details_file
def fetch_login_to_account_map(self) -> dict[str, str]:
conn = self._require_conn()
with conn.cursor() as cur:
_ = cur.execute("SELECT login_name, account_id FROM aps_customer")
rows = cur.fetchall()
login_to_account: dict[str, str] = {}
for row in rows:
login_name = safe_str(row.get("login_name"), 128)
account_id = safe_str(row.get("account_id"), 32)
if login_name and account_id:
login_to_account[login_name] = account_id
return login_to_account
@staticmethod
def normalize_customer_record(raw: JsonDict) -> JsonDict | None:
account_id = safe_str(raw.get("accountId"), 32)
@@ -906,20 +954,38 @@ class APSSyncer:
self.stats["bills"] += 1
# ---- Main sync entry ----
def sync_from_json(self, data_dir: str, incremental: bool = False) -> StatsDict:
def sync_from_json(self, data_dir: str, incremental: bool = False, sync_target: str = SYNC_TARGET_ALL) -> StatsDict:
start = datetime.now()
customers_file, orders_file, order_details_file, bills_file, customer_details_file = self.resolve_data_files(data_dir)
logger.info("Loading source files from %s%s", data_dir, " (增量模式)" if incremental else "")
normalized_sync_target = normalize_sync_target(sync_target)
customers_file, orders_file, order_details_file, bills_file, customer_details_file = self.resolve_data_files(
data_dir,
normalized_sync_target,
)
logger.info(
"Loading source files from %s%s%s",
data_dir,
" (增量模式)" if incremental else "",
"" if normalized_sync_target == SYNC_TARGET_ALL else f" (sync_target={normalized_sync_target})",
)
raw_customers: JsonList = []
raw_orders: JsonList = []
raw_order_details: JsonList = []
raw_bills: JsonList = []
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_CUSTOMER}:
raw_customers = self.load_json_records(customers_file)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_ORDER}:
raw_orders = self.load_json_records(orders_file)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_ORDERDETAILS}:
raw_order_details = self.load_json_records(order_details_file)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_BILLS}:
raw_bills = self.load_json_records(bills_file)
raw_customers = self.load_json_records(customers_file)
raw_orders = self.load_json_records(orders_file)
raw_order_details = self.load_json_records(order_details_file)
raw_bills = self.load_json_records(bills_file)
raw_customer_details: JsonList = []
try:
if customer_details_file.exists():
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_CUSTOMER} and customer_details_file.exists():
raw_customer_details = self.load_json_records(customer_details_file)
else:
elif normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_CUSTOMER}:
logger.info("Optional file missing, skip customer details: %s", customer_details_file)
except Exception as e:
logger.warning("Failed to load optional customer details file %s: %s", customer_details_file, e)
@@ -941,27 +1007,33 @@ class APSSyncer:
customers: JsonList = []
skipped_customers = 0
for raw in raw_customers:
c = self.normalize_customer_record(raw)
if not c:
skipped_customers += 1
continue
customers.append(c)
self.upsert_customer(c)
self.insert_snapshot(c, billing_month, captured_at)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_CUSTOMER}:
for raw in raw_customers:
c = self.normalize_customer_record(raw)
if not c:
skipped_customers += 1
continue
customers.append(c)
self.upsert_customer(c)
self.insert_snapshot(c, billing_month, captured_at)
if skipped_customers:
logger.info("Skipped %d invalid customer rows", skipped_customers)
login_to_account = build_login_to_account_map(customers)
if not login_to_account and normalized_sync_target in {SYNC_TARGET_ORDER, SYNC_TARGET_BILLS}:
login_to_account = self.fetch_login_to_account_map()
logger.info("Resolved %d customer login_name -> account_id mappings", len(login_to_account))
if raw_customer_details:
self.update_customer_details(raw_customer_details, billing_month)
self.upsert_orders(raw_orders, login_to_account)
self.upsert_order_details(raw_order_details)
self.sync_bills(raw_bills, login_to_account, incremental=incremental)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_ORDER}:
self.upsert_orders(raw_orders, login_to_account)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_ORDERDETAILS}:
self.upsert_order_details(raw_order_details)
if normalized_sync_target in {SYNC_TARGET_ALL, SYNC_TARGET_BILLS}:
self.sync_bills(raw_bills, login_to_account, incremental=incremental)
# Log sync
duration = (datetime.now() - start).total_seconds()
@@ -1051,10 +1123,17 @@ def main() -> None:
default=False,
help="仅查询 aps_bill 中最新的 consumption_time 并输出",
)
_ = parser.add_argument(
"--sync-target",
choices=sorted(VALID_SYNC_TARGETS),
default=SYNC_TARGET_ALL,
help="选择同步对象: all/customer/order/orderdetails/bills",
)
args = parser.parse_args()
data_dir = cast(str, args.dir)
incremental = cast(bool, args.incremental)
latest_bill_consumption_time = cast(bool, args.latest_bill_consumption_time)
sync_target = cast(str, args.sync_target)
syncer = APSSyncer(db_config=DB_CONFIG)
if latest_bill_consumption_time:
@@ -1066,7 +1145,7 @@ def main() -> None:
return
finally:
syncer.close()
_ = syncer.sync_from_json(data_dir, incremental=incremental)
_ = syncer.sync_from_json(data_dir, incremental=incremental, sync_target=sync_target)
if __name__ == "__main__":