import csv
import re
import html
import sys
import json
import requests

SITEMAP = "/home/bowerybay/public_html/vrp-sitemap.xml"
OUTDIR = "/home/bowerybay/public_html/wilgus-automation"
MASTER_JSON = OUTDIR + "/wilgus-properties-master.json"
META_CSV = OUTDIR + "/wilgus-meta-hotel-catalog.csv"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

META_FIELDS = [
    "id", "brand", "description", "title", "link",
    "address.addr1", "address.addr2", "address.city", "address.region",
    "address.postal_code", "address.country", "latitude", "longitude",
    "neighborhood[0]", "hotel_category", "video[0].url",
    "custom_label_0", "custom_label_1", "custom_label_2",
    "custom_label_3", "custom_label_4"
]

def get_attr(tag, name):
    m = re.search(r'\b' + re.escape(name) + r'="([^"]*)"', tag, re.I)
    return html.unescape(m.group(1)).strip() if m else ""

def clean_description(raw):
    raw = re.sub(r"<br\s*/?>", "\n", raw, flags=re.I)
    raw = re.sub(r"</p\s*>", "\n", raw, flags=re.I)
    raw = re.sub(r"<[^>]+>", " ", raw)
    raw = html.unescape(raw)
    raw = re.sub(r"[ \t]+", " ", raw)
    raw = re.sub(r"\s*\n\s*", "\n", raw)
    raw = re.sub(r"\n{2,}", "\n", raw)
    return raw.strip()

def meta_description(text, limit=5000):
    if len(text) <= limit:
        return text
    cut = text[:limit]
    last_space = cut.rfind(" ")
    if last_space > limit - 200:
        cut = cut[:last_space]
    return cut.rstrip()

def main():
    with open(SITEMAP, encoding="utf-8") as f:
        xml = f.read()

    urls = sorted(set(re.findall(
        r"<loc>(https://www\.wilgusassociates\.com/vrp/unit/[^<]+)</loc>", xml
    )))
    print(f"Found {len(urls)} property URLs in sitemap.")

    session = requests.Session()
    session.headers.update(HEADERS)
    properties = []

    for n, url in enumerate(urls, 1):
        print(f"[{n}/{len(urls)}] {url}", flush=True)
        try:
            response = session.get(url, timeout=30)
            response.raise_for_status()
            text = response.text
        except Exception as exc:
            print(f"  ERROR fetching page: {exc}", file=sys.stderr)
            continue

        m = re.search(r'<div\b[^>]*\bid="unit-data"[^>]*>', text, re.I | re.S)
        if not m:
            print("  ERROR: unit-data not found", file=sys.stderr)
            continue

        tag = m.group(0)
        code = get_attr(tag, "data-unit-property-code")
        pid = code.rsplit("-", 1)[-1] if code else ""

        dm = re.search(r'<div\s+id="description"[^>]*>(.*?)</div>', text, re.I | re.S)
        full_description = clean_description(dm.group(1)) if dm else ""

        prop = {
            "id": pid,
            "property_code": code,
            "site_unit_id": get_attr(tag, "data-unit-id"),
            "name": get_attr(tag, "data-unit-name"),
            "property_type": get_attr(tag, "data-unit-type"),
            "slug": get_attr(tag, "data-unit-slug"),
            "address1": get_attr(tag, "data-unit-address1"),
            "address2": get_attr(tag, "data-unit-address2"),
            "city": get_attr(tag, "data-unit-city"),
            "state": get_attr(tag, "data-unit-state"),
            "postal_code": get_attr(tag, "data-unit-zip"),
            "country": "US",
            "latitude": get_attr(tag, "data-unit-latitude"),
            "longitude": get_attr(tag, "data-unit-longitude"),
            "bedrooms": get_attr(tag, "data-unit-beds"),
            "bathrooms": get_attr(tag, "data-unit-baths"),
            "sleeps": get_attr(tag, "data-unit-sleeps"),
            "description": full_description,
            "listing_url": url,
            "video_url": f"https://bowerybay.com/wilgus-videos/{pid}.mp4",
        }
        properties.append(prop)

    with open(MASTER_JSON, "w", encoding="utf-8") as f:
        json.dump(properties, f, ensure_ascii=False, indent=2)

    rows = []
    for p in properties:
        rows.append({
            "id": p["id"],
            "brand": "Wilgus Associates",
            "description": meta_description(p["description"]),
            "title": p["name"],
            "link": p["listing_url"],
            "address.addr1": p["address1"],
            "address.addr2": p["address2"],
            "address.city": p["city"],
            "address.region": p["state"],
            "address.postal_code": p["postal_code"],
            "address.country": p["country"],
            "latitude": p["latitude"],
            "longitude": p["longitude"],
            "neighborhood[0]": p["city"],
            "hotel_category": p["property_type"],
            "video[0].url": p["video_url"],
            "custom_label_0": f'{p["bedrooms"]} Bedrooms' if p["bedrooms"] else "",
            "custom_label_1": f'{p["bathrooms"]} Bathrooms' if p["bathrooms"] else "",
            "custom_label_2": f'Sleeps {p["sleeps"]}' if p["sleeps"] else "",
            "custom_label_3": p["city"],
            "custom_label_4": p["property_type"],
        })

    with open(META_CSV, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.DictWriter(f, fieldnames=META_FIELDS)
        writer.writeheader()
        writer.writerows(rows)

    print(f"\nMaster data: {MASTER_JSON}")
    print(f"Meta catalog: {META_CSV}")
    print(f"Properties written: {len(properties)}")

if __name__ == "__main__":
    main()