lucky_shop/scripts/audit_lang_bundle.py

"""Full $lang vs bundle audit.

Steps:
  1. Load /api/lang/bundle (lang-bundle.json) tree
  2. Extract every $lang('xxx'[, ...]) first-arg string from .vue/.js files
  3. Locate each key in bundle tree (and a small set of structural fallbacks)
  4. Classify: FOUND / MISSING / SYNTAX_ERROR
  5. For MISSING: also check local lang files (zh-cn, en-us) to determine
     whether the key exists locally but bundle missed it
"""
import json, os, re, subprocess
from collections import defaultdict, Counter

ROOT = r'D:\projects\shop-projects\frontend'
BUNDLE_PATH = os.path.join(ROOT, 'lang-bundle.json')
NODE_SCRIPT = os.path.join(ROOT, 'scripts', 'audit_lang_bundle.js')

# ---------- 1. Load bundle ----------
with open(BUNDLE_PATH, 'r', encoding='utf-8') as f:
    bundle = json.load(f)
tree = bundle['data']['tree']
print(f'[bundle] version={bundle["data"].get("version")} lang={bundle["data"].get("lang")} source={bundle["data"].get("source")}')

# Flatten bundle keys (leaf string values)
bundle_leaves = set()
bundle_nodes = set()  # intermediate nodes too, for fallback lookups

def walk(obj, prefix=''):
    if isinstance(obj, dict):
        if prefix:
            bundle_nodes.add(prefix)
        for k, v in obj.items():
            nk = f'{prefix}.{k}' if prefix else k
            if isinstance(v, dict):
                walk(v, nk)
            else:
                bundle_leaves.add(nk)
    elif prefix:
        bundle_leaves.add(prefix)
walk(tree)
print(f'[bundle] {len(bundle_leaves)} leaf keys, {len(bundle_nodes)} intermediate nodes')

# ---------- 2. Extract $lang() calls ----------
SKIP_DIRS = {'node_modules', '.unpackage', 'unpackage', '.git', 'dist', 'stub-app-projects'}
all_files = []
for dirpath, dirnames, filenames in os.walk(ROOT):
    if any(s in dirpath for s in SKIP_DIRS):
        continue
    for fn in filenames:
        if fn.endswith(('.vue', '.js')):
            all_files.append(os.path.join(dirpath, fn))

# Match $lang('key'[, ...]) - first string arg, single OR double quotes
# $lang() can span many lines but first arg is generally on same line
call_re = re.compile(r"\$lang\(\s*(['\"])([^'\"]+)\1")
all_calls = []
for fp in all_files:
    rel = os.path.relpath(fp, ROOT)
    with open(fp, 'r', encoding='utf-8') as fh:
        try:
            content = fh.read()
        except UnicodeDecodeError:
            continue
    for m in call_re.finditer(content):
        all_calls.append({'file': rel, 'key': m.group(2), 'pos': m.start()})

print(f'[code] {len(all_calls)} $lang calls across {len(set(c["file"] for c in all_calls))} files')
unique_keys = sorted(set(c['key'] for c in all_calls))
print(f'[code] {len(unique_keys)} unique keys called')

# ---------- 3. Classify ----------
found, missing = [], []
for k in unique_keys:
    if k in bundle_leaves:
        found.append(k)
    else:
        missing.append(k)

print(f'[match] FOUND={len(found)}  MISSING={len(missing)}')

# ---------- 4. For MISSING keys: check local lang files too ----------
print()
print('Loading local lang files via Node...')
def load_locale(locale):
    res = subprocess.run(['node', NODE_SCRIPT, locale], cwd=ROOT,
                         capture_output=True, text=True, encoding='utf-8')
    if res.returncode != 0:
        print(f'  node {locale} stderr: {res.stderr[:200]}')
        return {}
    parsed = json.loads(res.stdout)
    cleaned = {}
    for full, langobj in parsed.items():
        rel = os.path.relpath(full, os.path.join(ROOT, 'lang', locale))
        rel_no_ext = rel[:-3] if rel.endswith('.js') else rel
        cleaned[rel_no_ext] = langobj
    return cleaned

zh_cn = load_locale('zh-cn')
en_us = load_locale('en-us')
print(f'  zh-cn: {len(zh_cn)} files; en-us: {len(en_us)} files')

def lookup_full(key, data):
    """Walk dotted segments in nested map."""
    cur = data
    for seg in key.split('.'):
        if not isinstance(cur, dict) or seg not in cur:
            return None
        cur = cur[seg]
    return cur if isinstance(cur, str) else None

def local_lookup(key, locale_map):
    arr = key.split('.')
    last = arr[-1]
    fname_segments = '/'.join(arr[:-1])
    matches = []
    # 1) Short key in flat page file: e.g. fenxiao/bill.js exports {billNoPrefix}
    for fname, data in locale_map.items():
        if not isinstance(data, dict):
            continue
        if (fname.endswith('/' + fname_segments) or fname == fname_segments) and \
           last in data and isinstance(data[last], str):
            matches.append(('short:' + last, fname, data[last]))
    # 2) Full dotted in any nested file
    for fname, data in locale_map.items():
        if not isinstance(data, dict):
            continue
        v = lookup_full(key, data)
        if isinstance(v, str):
            matches.append(('nested:' + key, fname, v))
            break
    return matches

def classify_missing(key):
    """Returns (category, evidence). category in {LOCALLY_DEFINED, SYNTAX_BUG, TRULY_MISSING}"""
    # Syntax error heuristics
    if key.endswith('.') or key in ('key', 'title', 'xxx'):
        return 'SYNTAX_BUG', 'trailing dot or generic identifier'
    if 'xxx' in key.lower() and not (key.startswith('xxx') and key.count('.') == 0):
        # 'components.foo.title' / 'components-diy.xxx' style placeholders in mixin
        return 'SYNTAX_BUG', "placeholder/example string (e.g. 'components.foo.title')"
    # Check local files
    zh = local_lookup(key, zh_cn)
    en = local_lookup(key, en_us)
    if zh or en:
        # Capture the actual found path in local file for evidence
        ev = ''
        if zh:
            ev += f'zh-cn: {zh[0][1]} {zh[0][0]}={zh[0][2]!r}'
        if en:
            ev += (' | ' if ev else '') + f'en-us: {en[0][1]} {en[0][0]}'
        return 'LOCALLY_DEFINED', ev
    return 'TRULY_MISSING', 'not in zh-cn / en-us / bundle'

cat_counts = Counter()
locally_defined = []
syntax_bugs = []
truly_missing = []
for k in missing:
    cat, ev = classify_missing(k)
    cat_counts[cat] += 1
    usage = [c for c in all_calls if c['key'] == k]
    entry = {
        'key': k, 'category': cat, 'evidence': ev,
        'usage_count': len(usage), 'files': sorted(set(c['file'] for c in usage))[:6],
    }
    if cat == 'LOCALLY_DEFINED':
        locally_defined.append(entry)
    elif cat == 'SYNTAX_BUG':
        syntax_bugs.append(entry)
    else:
        truly_missing.append(entry)

print()
print('=== Classification of missing keys ===')
for cat, n in cat_counts.most_common():
    print(f'  {cat}: {n}')

# ---------- 5. Inspect a few interesting key paths in bundle ----------
# Find for each "locally defined" key, the actual structure in bundle (closest match)
def closest_in_bundle(key):
    """Return bundle path segment that contains parts of key."""
    parts = key.split('.')
    cur = tree
    best = tree
    path = []
    for p in parts:
        if isinstance(cur, dict):
            cand = cur.get(p)
            if cand is None:
                # Try camelCase
                cc = re.sub(r'[-_]([a-z])', lambda m: m.group(1).upper(), p)
                cand = cur.get(cc)
                if cand is not None:
                    path.append(cc)
                    cur = cand
                    continue
            if cand is not None:
                path.append(p)
                cur = cand
            else:
                return '.'.join(path), type(cur).__name__
        else:
            return '.'.join(path), type(cur).__name__
    return '.'.join(path), type(cur).__name__

print()
print('=== For LOCALLY_DEFINED keys: real bundle structure check ===')
for e in locally_defined[:10]:
    actual = closest_in_bundle(e['key'])
    print(f'  $lang({e["key"]})')
    print(f'      call: {e["files"][0]}')
    print(f'      bundle partial walk stopped at: {actual[0]} ({actual[1]})')

# ---------- 6. Save report ----------
report = {
    'bundle_version': bundle['data'].get('version'),
    'bundle_lang': bundle['data'].get('lang'),
    'bundle_source': bundle['data'].get('source'),
    'bundle_leaves': len(bundle_leaves),
    'code_calls_total': len(all_calls),
    'code_unique_keys': len(unique_keys),
    'found_in_bundle': len(found),
    'missing_summary': dict(cat_counts),
    'missing_locally_defined': locally_defined,
    'missing_syntax_bugs': syntax_bugs,
    'missing_truly_missing': truly_missing,
}
out_path = os.path.join(ROOT, 'scripts', 'lang_audit_report.json')
with open(out_path, 'w', encoding='utf-8') as f:
    json.dump(report, f, ensure_ascii=False, indent=2)
print(f'\nReport saved to {out_path}')