"""Read-only production inventory. Requires lxml; stores source HTML outside public/.
Run with the workspace Python runtime. Does not write to WordPress.
"""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import Request, urlopen
from urllib.parse import urljoin, urlsplit
import csv, hashlib, json, re, time, sys
from lxml import html, etree

ROOT = Path(__file__).resolve().parents[1]
WORK = ROOT / 'work/production-audit'
WORK.mkdir(parents=True, exist_ok=True)
ORIGIN = 'https://portaldistribuidora.com.br'

def fetch(url):
    for attempt in range(3):
        try:
            with urlopen(Request(url, headers={'User-Agent': 'PortalMigrationAudit/1.0'}), timeout=35) as response:
                return response.status, response.url, response.read().decode('utf-8', 'replace')
        except Exception as error:
            if attempt == 2: return getattr(error, 'code', 0), url, str(error)
            time.sleep(1 + attempt)

def sitemap(url):
    status, _, body = fetch(url)
    if status != 200: raise RuntimeError(f'Sitemap failed: {url}: {status}')
    (WORK / urlsplit(url).path.strip('/')).write_text(body)
    root = etree.fromstring(body.encode())
    entries = root.findall('{*}sitemap')
    if entries:
        return [item for entry in entries for item in sitemap(entry.find('{*}loc').text)]
    return [{'url': entry.find('{*}loc').text, 'sitemap': url, 'lastmod': entry.findtext('{*}lastmod', '')} for entry in root.findall('{*}url')]

def clean(value): return re.sub(r'\s+', ' ', value or '').strip()

def audit(entry):
    url = entry['url']
    status, final, body = fetch(url)
    filename = hashlib.sha256(url.encode()).hexdigest()[:16] + '.html'
    (WORK / filename).write_text(body)
    result = {**entry, 'status': status, 'finalUrl': final, 'sourceFile': filename}
    if status != 200: return result
    if urlsplit(url).path.endswith('.kml') and '<kml' in body:
        return {**result, 'asset': True, 'contentText': body, 'title': 'Mapa KML da Portal'}
    document = html.fromstring(body.encode())
    title = document.xpath('//title/text()')
    description = document.xpath('//meta[@name="description"]/@content')
    canonical = document.xpath('//link[@rel="canonical"]/@href')
    result.update(title=clean(' '.join(title)), description=clean(' '.join(description)), canonical=canonical[0] if canonical else '', h1=[clean(x.text_content()) for x in document.xpath('//h1')], analyticsIds=sorted(set(re.findall(r'\b(?:G-[A-Z0-9]{5,15}|GTM-[A-Z0-9]+)\b', body))))
    candidates = document.xpath('//main | //*[@data-elementor-type="wp-page"] | //*[@data-elementor-type="single-post"] | //*[@data-elementor-type="single"] | //*[@data-elementor-type="archive"]')
    main = max(candidates, key=lambda x:len(x.text_content())) if candidates else (document.find('body') if document.find('body') is not None else document)
    for node in main.xpath('.//script | .//style | .//nav | .//header | .//footer | .//form | .//noscript | .//*[@data-elementor-type="header"] | .//*[@data-elementor-type="footer"]'):
        if node.getparent() is not None: node.drop_tree()
    result['blocked'] = 'Please wait while your request is being verified' in body
    result['contentText'] = clean(main.text_content())
    result['headings'] = [clean(x.text_content()) for x in main.xpath('.//h2 | .//h3')]
    result['links'] = sorted(set(urljoin(final, x) for x in main.xpath('.//a/@href') if x and not x.startswith(('javascript:', 'mailto:', 'tel:'))))
    result['images'] = [{'src':urljoin(final, x.get('src','')), 'alt':x.get('alt',''), 'width':x.get('width',''), 'height':x.get('height','')} for x in main.xpath('.//img[@src]')]
    result['mainHtml'] = html.tostring(main, encoding='unicode')
    return result

if __name__ == '__main__':
    if '--resume' in sys.argv:
        output = json.loads((WORK/'inventory.json').read_text())
        for index, item in enumerate(output['pages']):
            if 'One moment' in item.get('title', '') or item['status'] != 200:
                time.sleep(1.2)
                result = audit({k:item[k] for k in ['url','sitemap','lastmod']})
                output['pages'][index] = result
                (WORK/'inventory.json').write_text(json.dumps(output, ensure_ascii=False, indent=2))
                print(result['status'], result.get('title','')[:70], flush=True)
                if result.get('blocked'): print('Verification page remains; stopping retry.'); break
        sys.exit(0)
    entries = {x['url']:x for x in sitemap(ORIGIN+'/sitemap_index.xml')}
    # Historical URLs remain part of the comparison even when absent from today's sitemap.
    with (ROOT.parent/'auditoria-portal-paginas.csv').open() as f:
        for row in csv.DictReader(f): entries.setdefault(row['url'], {'url':row['url'], 'sitemap':'historical-audit', 'lastmod':''})
    results = []
    with ThreadPoolExecutor(max_workers=4) as pool:
        for result in pool.map(audit, entries.values()):
            results.append(result)
            if len(results)%20 == 0: print(f'Audited {len(results)}/{len(entries)}', flush=True)
    output = {'origin':ORIGIN, 'auditedAt':datetime.now(timezone.utc).isoformat(), 'pages':results}
    (WORK/'inventory.json').write_text(json.dumps(output, ensure_ascii=False, indent=2))
    print(json.dumps({'pages':len(results), 'status':{str(status):sum(x['status']==status for x in results) for status in set(x['status'] for x in results)}, 'analyticsIds':sorted(set(v for x in results for v in x.get('analyticsIds',[])))},ensure_ascii=False))