import csv
import re
import html
import sys
import requests

SITEMAP = "/home/bowerybay/public_html/vrp-sitemap.xml"
OUTPUT = "/home/bowerybay/public_html/wilgus-automation/wilgus-catalog-preview.csv"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

FIELDS = [
    "id", "brand", "description", "title", "link",
    "address.addr1", "address.city", "address.region", "address.country",
    "latitude", "longitude", "neighborhood[0]", "video[0].url"
]

def get_attr(tag, name):
    m = re.search(r'\b' + re.escape(name) + r'="([^"]*)"', tag, re.I)
    return html.unescape(m.group(1)).strip() if m else ""

def clean_description(raw):
    raw = re.sub(r"<br\s*/?>", " ", raw, flags=re.I)
    raw = re.sub(r"</p\s*>", " ", raw, flags=re.I)
    raw = re.sub(r"<[^>]+>", " ", raw)
    return re.sub(r"\s+", " ", html.unescape(raw)).strip()

def main():
    with open(SITEMAP, encoding="utf-8") as f:
        xml = f.read()

    urls = sorted(set(re.findall(
        r"<loc>(https://www\.wilgusassociates\.com/vrp/unit/[^<]+)</loc>", xml
    )))
    print(f"Found {len(urls)} property URLs in sitemap.")

    session = requests.Session()
    session.headers.update(HEADERS)
    rows = []

    for n, url in enumerate(urls, 1):
        print(f"[{n}/{len(urls)}] {url}", flush=True)
        try:
            response = session.get(url, timeout=30)
            response.raise_for_status()
            text = response.text
        except Exception as exc:
            print(f"  ERROR fetching page: {exc}", file=sys.stderr)
            continue

        m = re.search(r'<div\b[^>]*\bid="unit-data"[^>]*>', text, re.I | re.S)
        if not m:
            print("  ERROR: unit-data not found", file=sys.stderr)
            continue

        tag = m.group(0)
        code = get_attr(tag, "data-unit-property-code")
        pid = code.rsplit("-", 1)[-1] if code else ""

        dm = re.search(r'<div\s+id="description"[^>]*>(.*?)</div>', text, re.I | re.S)
        description = clean_description(dm.group(1)) if dm else ""

        rows.append({
            "id": pid,
            "brand": "Wilgus Associates",
            "description": description,
            "title": get_attr(tag, "data-unit-name"),
            "link": url,
            "address.addr1": get_attr(tag, "data-unit-address1"),
            "address.city": get_attr(tag, "data-unit-city"),
            "address.region": get_attr(tag, "data-unit-state"),
            "address.country": "US",
            "latitude": get_attr(tag, "data-unit-latitude"),
            "longitude": get_attr(tag, "data-unit-longitude"),
            "neighborhood[0]": get_attr(tag, "data-unit-city"),
            "video[0].url": f"https://bowerybay.com/wilgus-videos/{pid}.mp4",
        })

    with open(OUTPUT, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(rows)

    print(f"\nWrote {len(rows)} rows to {OUTPUT}")

if __name__ == "__main__":
    main()