#!/usr/bin/env python3
"""
SF describe → nexuscrm pack ObjectSpec extractor.

Input:  directory of Salesforce REST `describeSObject` JSON dumps
        (one file per object, named <ApiName>.json, top-level shape:
         {"status":0,"result":{...describe...}})

Output: per-object pack spec files + manifest components snippet
        + conversion-loss report capturing information that the current
        nexuscrm pack ObjectSpec/FieldSpec schema cannot represent.

Design notes:
  * "Faithful but mark-and-fail" — drop only what is provably non-standard
    (custom: true dev samples, Id primary key, deprecatedAndHidden); keep
    everything else, including system fields like CreatedDate/OwnerId, so
    that L0 install/verify behavior under realistic input is exercised.
  * Information that the pack schema cannot carry (length, scale, int/long
    distinction, address/base64 composite types) is recorded in
    conversion-loss.md, not silently dropped. Facts that describeSObject does
    not expose (sharingModel, enableHierarchySharing) are intentionally not
    emitted; the install-pack apply path falls back to the L0 substrate
    default for these fields, keeping a single read path with bootstrap.
  * One file per object → preserves component-level granularity for
    manifest-side dependsOn topology testing.
"""
from __future__ import annotations

import json
import sys
from collections import defaultdict
from pathlib import Path
from typing import Any

DESCRIBE_DIR = Path("/tmp/sfdescribe/describe")
OUT_ROOT = Path("/tmp/sfmeta/extracted")
OUT_OBJECTS = OUT_ROOT / "objects"
OUT_MANIFEST_SNIPPET = OUT_ROOT / "manifest_components.json"
OUT_LOSS_REPORT = OUT_ROOT / "conversion-loss.md"

# Map describe field types → nexuscrm constants.SchemaFieldType vocabulary.
# Anything not in this table either drops (Id) or is recorded as a loss
# (address, base64).
TYPE_MAP: dict[str, str] = {
    "string":         "string",
    "textarea":       "textarea",
    "email":          "email",
    "phone":          "phone",
    "url":            "url",
    "boolean":        "boolean",
    "picklist":       "picklist",
    "multipicklist":  "multipicklist",
    "combobox":       "picklist",   # SF legacy alias
    "currency":       "currency",
    "percent":        "percent",
    "double":         "double",
    "int":            "double",     # LOSS: pack has no int type
    "long":           "double",     # LOSS: pack has no long type
    "date":           "date",
    "datetime":       "datetime",
    "reference":      "reference",
    # "address" / "base64" intentionally absent → caught as loss
    # "id" handled by drop_field()
}

# Platform-managed objects: per nexuscrm-platform-architecture.md §3,
# generic admin/identity objects belong to L0 (platform binary owns them
# via platform-core pack at init-l0). L1 packs must not redeclare them.
# Including them triggers "ownerless ... not same-pack retry-compatible".
PLATFORM_MANAGED_OBJECTS = {
    "User", "Profile", "Group", "UserRole",
    # Hierarchies / shares are platform too:
    "PermissionSet", "PermissionSetGroup",
}


# Platform-managed system columns (mirrors
# backend/internal/infrastructure/persistence/schema_system_columns.go
# GetStandardSystemColumns + FieldName addition). L0 auto-injects these on every
# object; L1 packs must not redeclare them or install fails with
# "conflicting relationship declaration" / duplicate field errors.
PLATFORM_MANAGED_FIELDS = {
    "Id", "OwnerId", "CreatedById", "LastModifiedById",
    "CreatedDate", "LastModifiedDate", "SystemModstamp", "IsDeleted",
    # Name is auto-injected for objects with isNameField semantics; we still
    # let the extractor emit it because nameField presence varies per object
    # and the pack reader expects an explicit isNameField marker.
    # SF surface fields that are derived/internal:
    "LastViewedDate", "LastReferencedDate",
}


def drop_field(fld: dict[str, Any]) -> str | None:
    """Return reason if field should be dropped from pack output, else None."""
    if fld["name"] == "Id" and fld["type"] == "id":
        return "primary-key (L0 owns _System_Object.Id)"
    if fld["name"] in PLATFORM_MANAGED_FIELDS:
        return f"platform-managed system field (L0 auto-injects via _System_*): {fld['name']}"
    if fld.get("custom", False):
        return f"dev-org sample custom field (custom=true): {fld['name']}"
    if fld.get("deprecatedAndHidden", False):
        return f"deprecatedAndHidden: {fld['name']}"
    return None


def is_master_detail(fld: dict[str, Any]) -> bool:
    """SF describe encodes master-detail via non-null relationshipOrder."""
    return fld.get("type") == "reference" and fld.get("relationshipOrder") is not None


def map_field(fld: dict[str, Any], losses: list[str], obj_name: str,
              child_rel_index: dict[tuple[str, str], dict[str, str]] | None = None) -> dict[str, Any] | None:
    """Convert a describe field record to a pack FieldSpec dict.

    Records type/attr losses into `losses` list with full context.
    Returns None if the field cannot be mapped (e.g., unmapped type) — caller
    decides whether to skip or include a marker entry.
    """
    sf_type = fld["type"]
    pack_type = TYPE_MAP.get(sf_type)
    if pack_type is None:
        losses.append(
            f"- `{obj_name}.{fld['name']}` SF type `{sf_type}` has no pack-side mapping → field omitted"
        )
        return None

    # Calculated formula: SF reports calc field's return type as `type`,
    # but pack semantics want type=formula + returnType=<calc result type>.
    if fld.get("calculatedFormula"):
        spec: dict[str, Any] = {
            "apiName":    fld["name"],
            "label":      fld["label"],
            "type":       "formula",
            "returnType": pack_type,
            "formula":    fld["calculatedFormula"],
        }
    else:
        spec = {
            "apiName": fld["name"],
            "label":   fld["label"],
            "type":    pack_type,
        }

    # required: nillable=False AND no defaultedOnCreate → user must provide
    if not fld.get("nillable", True) and not fld.get("defaultedOnCreate", False):
        spec["required"] = True

    if fld.get("nameField", False):
        spec["isNameField"] = True

    # createable / updateable: only emit when SF says false (most fields are
    # both true; clutter-reducing emission)
    if fld.get("createable", True) is False:
        spec["createable"] = False
    if fld.get("updateable", True) is False:
        spec["updateable"] = False

    # Default value — but multipicklist/textarea map to JSON/TEXT columns
    # in L0 which MySQL forbids DEFAULT on. SF allows defaults on these;
    # nexuscrm cannot honor them. Record as loss and skip.
    default_formula = fld.get("defaultValueFormula")
    default_value = fld.get("defaultValue")
    has_literal_default = default_value is not None and default_value != ""
    if default_formula or has_literal_default:
        if sf_type in ("multipicklist", "textarea"):
            losses.append(
                f"- `{obj_name}.{fld['name']}` SF {sf_type} default='{default_formula or default_value}' dropped — L0 maps to JSON/TEXT, MySQL forbids DEFAULT on those types"
            )
        else:
            if default_formula:
                spec["default"] = default_formula
            elif isinstance(default_value, bool):
                spec["default"] = "1" if default_value else "0"
            elif has_literal_default:
                spec["default"] = str(default_value)

    # Reference / lookup
    if pack_type == "reference":
        ref_to = fld.get("referenceTo") or []
        if ref_to:
            spec["referenceTo"] = list(ref_to)
        if fld.get("relationshipName"):
            spec["relationshipName"] = fld["relationshipName"]
        if len(ref_to) > 1:
            spec["isPolymorphic"] = True

        # Authoritative child-side relationship name. SF publishes one entry
        # per parent in `parent.describe.childRelationships[]`; we index it
        # in main() before invoking map_object/map_field. For polymorphic
        # FKs SF publishes the same name across parents (verified in
        # /tmp/sfdescribe spot-checks); we take any populated entry and
        # surface a loss if real-world divergence ever appears. For FKs
        # whose parent objects we have no describe dump for (User/Group/
        # RecordType/etc.) the index has no entry — leave the field unset
        # rather than guess; PR-side validators reject blanks separately.
        if child_rel_index is not None:
            key = (obj_name.lower(), fld["name"].lower())
            published = child_rel_index.get(key, {})
            populated = [v for v in published.values() if v]
            if populated:
                first = populated[0]
                if any(v != first for v in populated):
                    losses.append(
                        f"- `{obj_name}.{fld['name']}` polymorphic FK has divergent childRelationshipName across parents: {published}"
                    )
                spec["childRelationshipName"] = first
            else:
                missing_parents = [
                    r for r in ref_to
                    if (obj_name.lower(), fld["name"].lower()) not in child_rel_index
                    or r.lower() not in published
                ]
                losses.append(
                    f"- `{obj_name}.{fld['name']}` no authoritative childRelationshipName (parents missing describe or SF published null): {missing_parents}"
                )

    # Picklist values: keep ALL values (active and inactive) per faithfulness;
    # mark inactive via tags so L0 ingestion tests see the variety.
    if pack_type in ("picklist", "multipicklist"):
        opts = []
        default_val = None
        for pv in fld.get("picklistValues", []):
            opt = {"value": pv["value"], "label": pv["label"]}
            tags = []
            if pv.get("active", True) is False:
                tags.append("inactive")
            if tags:
                opt["tags"] = tags
            opts.append(opt)
            if pv.get("defaultValue", False):
                default_val = pv["value"]
        if opts:
            spec["options"] = opts
            if default_val and "default" not in spec:
                # Multipicklist defaults can't be applied at DDL layer
                # (L0 maps multipicklist → JSON column; MySQL forbids
                # DEFAULT on JSON/TEXT). Track loss and skip default.
                if pack_type == "multipicklist":
                    losses.append(
                        f"- `{obj_name}.{fld['name']}` SF multipicklist default='{default_val}' dropped (picklist-value branch) — L0 JSON column, MySQL forbids DEFAULT"
                    )
                else:
                    spec["default"] = default_val
        else:
            # SF picklists with no values (Global Value Set / restricted picklist
            # not exposed via describe) — L0 currently rejects empty options;
            # downgrade to string and record loss so install can proceed.
            spec["type"] = "string"
            losses.append(
                f"- `{obj_name}.{fld['name']}` SF picklist with empty picklistValues → downgraded to `string` (L0 forbids picklist with zero options)"
            )

    # Dependent picklist
    if fld.get("controllerName"):
        spec["controllingField"] = fld["controllerName"]

    # ---- losses (information present in describe, not in pack schema) ----
    # length / scale / precision: only meaningful for sized types
    if sf_type in ("string", "textarea", "url", "phone", "email", "encryptedstring"):
        if fld.get("length") not in (None, 0):
            losses.append(
                f"- `{obj_name}.{fld['name']}` length={fld['length']} dropped — pack FieldSpec has no length"
            )
    if sf_type in ("double", "currency", "percent", "int", "long"):
        if fld.get("precision") not in (None, 0) or fld.get("scale") not in (None, 0):
            losses.append(
                f"- `{obj_name}.{fld['name']}` precision={fld.get('precision')} scale={fld.get('scale')} dropped — pack FieldSpec has no precision/scale"
            )
    if sf_type in ("int", "long"):
        losses.append(
            f"- `{obj_name}.{fld['name']}` SF type `{sf_type}` collapsed to pack `double`"
        )

    return spec


def map_object(describe: dict[str, Any], losses: list[str],
               child_rel_index: dict[tuple[str, str], dict[str, str]] | None = None) -> tuple[dict[str, Any], list[tuple[str, str]], set[str]]:
    """Convert one describe result block to a pack ObjectSpec dict.

    Returns (obj_spec, drop_log, master_detail_parents) where master_detail_parents
    is the set of parent object api-names this object's master-detail fields point to.
    """
    api = describe["name"]
    fields_out = []
    drop_log = []
    md_parents: set[str] = set()
    for fld in describe["fields"]:
        reason = drop_field(fld)
        if reason:
            drop_log.append((fld["name"], reason))
            continue
        spec = map_field(fld, losses, api, child_rel_index)
        if spec is not None:
            fields_out.append(spec)
        if is_master_detail(fld):
            for r in fld.get("referenceTo", []):
                if r != api:
                    md_parents.add(r)

    # SharingModel and EnableHierarchySharing are not extracted: describeSObject
    # does not expose org-wide-default. Pack declines authority on these facts;
    # install-pack apply path falls back to bootstrap.DefaultSharingModel
    # (Private for standard_object). Authoritative SF v66 values require Tooling
    # API queries on EntityDefinition or post-install L2 admin override.
    obj_spec: dict[str, Any] = {
        "apiName":                api,
        "keyPrefix":              describe.get("keyPrefix", ""),
        "label":                  describe["label"],
        "pluralLabel":            describe.get("labelPlural", ""),
        "tableType":              "standard_object",
        "category":               "crm",
        "description":            f"Salesforce standard {api} (v66.0 dev-org gold master)",
        "fields":                 fields_out,
    }

    # Object-level losses
    losses.append(
        f"- `{api}` sharingModel and enableHierarchySharing intentionally not extracted — describeSObject does not expose org-wide-default; L0 substrate fallback (`Private` for standard_object) applies on install"
    )
    if describe.get("recordTypeInfos"):
        rt_count = len(describe["recordTypeInfos"])
        master_only = (rt_count == 1 and describe["recordTypeInfos"][0].get("master", False))
        if not master_only:
            losses.append(
                f"- `{api}` has {rt_count} recordTypeInfos — not embedded in object_spec, must ship as separate `record_type` component"
            )
    return obj_spec, drop_log, md_parents


# System-managed foreign-key fields. They reference real objects, but the
# pack lifecycle does not require those targets to be applied earlier — these
# FKs are platform plumbing, not architectural component dependencies.
# Including them creates spurious cycles in the dependsOn graph (e.g. every
# object's OwnerId → User → User.CreatedById → User → ...).
SYSTEM_FK_FIELDS = {
    "OwnerId", "CreatedById", "LastModifiedById",
    "MasterRecordId", "RecordTypeId", "ManagerId",
    "DelegatedApproverId", "UserRoleId", "ProfileId",
}


def topo_sort(objs: list[dict[str, Any]], available: set[str],
              md_edges: dict[str, set[str]]) -> list[str]:
    """Return object apiNames in dependency-respecting order (parents first).

    Edges = master-detail relationships only (passed in via `md_edges`).
    Plain lookups are not load-order dependencies — they store typed FKs whose
    target may legally be NULL or applied later, just like SF's own pack
    install behavior.
    """
    deps = defaultdict(set)
    for nm in (o["apiName"] for o in objs):
        deps[nm] = set(md_edges.get(nm, set()))

    visited = set()
    order = []
    visiting = set()  # cycle guard
    cycles: list[tuple[str, str]] = []

    def visit(n: str):
        if n in visited:
            return
        if n in visiting:
            cycles.append((n, ",".join(sorted(visiting))))
            return
        visiting.add(n)
        for d in deps[n]:
            visit(d)
        visiting.remove(n)
        visited.add(n)
        order.append(n)

    for n in sorted(deps):
        visit(n)

    if cycles:
        print(f"WARN: {len(cycles)} dependsOn cycles detected:", file=sys.stderr)
        for n, path in cycles:
            print(f"  {n} reached during visit of {{{path}}}", file=sys.stderr)
    return order


def main() -> int:
    if not DESCRIBE_DIR.exists():
        print(f"missing describe dir: {DESCRIBE_DIR}", file=sys.stderr)
        return 1

    # Clean previous output so renamed/dropped objects (e.g. moves into
    # PLATFORM_MANAGED_OBJECTS) don't leave stale dirs behind for downstream
    # cp -R. Idempotent rebuild from describe input.
    if OUT_OBJECTS.exists():
        for child in OUT_OBJECTS.iterdir():
            if child.is_dir():
                for sub in child.iterdir():
                    sub.unlink()
                child.rmdir()
    OUT_OBJECTS.mkdir(parents=True, exist_ok=True)

    losses: list[str] = []
    drop_summary = defaultdict(int)
    objs_out: list[dict[str, Any]] = []
    md_edges: dict[str, set[str]] = {}
    # Exclude platform-managed objects from "available" so dependsOn/topology
    # never points at L0-owned objects (User, Profile, Group, …).
    available_names = {
        p.stem for p in DESCRIBE_DIR.glob("*.json")
        if p.stem not in PLATFORM_MANAGED_OBJECTS
    }

    # Build (child_obj_lower, fk_field_lower) → {parent_lower: relName} index
    # from each parent's describe.childRelationships[]. Used by map_field to
    # emit authoritative childRelationshipName per FK.
    child_rel_index: dict[tuple[str, str], dict[str, str]] = {}
    for jf in sorted(DESCRIBE_DIR.glob("*.json")):
        try:
            raw = json.loads(jf.read_text())
        except Exception:
            continue
        result = raw.get("result")
        if not isinstance(result, dict):
            continue
        parent = (result.get("name") or "").lower()
        if not parent:
            continue
        for cr in result.get("childRelationships", []) or []:
            rn = cr.get("relationshipName")
            if not rn:
                continue
            child = (cr.get("childSObject") or "").lower()
            field = (cr.get("field") or "").lower()
            if not child or not field:
                continue
            child_rel_index.setdefault((child, field), {})[parent] = rn

    for jf in sorted(DESCRIBE_DIR.glob("*.json")):
        raw = json.loads(jf.read_text())
        if "result" not in raw:
            losses.append(f"- `{jf.name}` no `result` key, skipped")
            continue
        obj_name = raw["result"]["name"]
        if obj_name in PLATFORM_MANAGED_OBJECTS:
            losses.append(
                f"- `{obj_name}` skipped — platform-managed object (L0 owns; per architecture §3 L1 packs must not redeclare)"
            )
            continue
        obj_spec, drop_log, md_parents = map_object(raw["result"], losses, child_rel_index)
        objs_out.append(obj_spec)
        md_edges[obj_spec["apiName"]] = {p for p in md_parents if p in available_names}
        for _, reason in drop_log:
            drop_summary[reason.split(":")[0].strip()] += 1

        # Single-element list to align with current spec_reader.go top-level shape
        out_path = OUT_OBJECTS / obj_spec["apiName"] / "spec.json"
        out_path.parent.mkdir(parents=True, exist_ok=True)
        out_path.write_text(json.dumps([obj_spec], indent=2, ensure_ascii=False) + "\n")

    # ---- manifest components snippet ----
    order = topo_sort(objs_out, available_names, md_edges)
    by_name = {o["apiName"]: o for o in objs_out}

    components = []
    for nm in order:
        comp = {
            "kind":          "object_spec",
            "businessKey":   f"object:{nm}",
            "manageability": "locked",
            "path":          f"objects/{nm}/spec.json",
        }
        # dependsOn = master-detail parents only (plain lookups are not
        # load-order requirements; they store typed FKs that can resolve later).
        parents = sorted(md_edges.get(nm, set()))
        if parents:
            comp["dependsOn"] = [
                {"kind": "object_spec", "businessKey": f"object:{p}"}
                for p in parents
            ]
        components.append(comp)

    OUT_MANIFEST_SNIPPET.write_text(
        json.dumps({"components": components}, indent=2, ensure_ascii=False) + "\n"
    )

    # ---- loss report ----
    out = ["# Conversion losses: SF describe → nexuscrm pack ObjectSpec",
           f"\nSource: `{DESCRIBE_DIR}` (32 v66.0 dev-org describe JSONs)",
           f"Output: `{OUT_OBJECTS}/<Obj>/spec.json`\n",
           "## Drops (intentional, not losses)\n"]
    for r, n in sorted(drop_summary.items(), key=lambda x: -x[1]):
        out.append(f"- {r}: {n} fields")
    out.append("\n## Information losses (pack schema cannot represent)\n")
    out.extend(losses)
    out.append("\n## Loss summary by category\n")
    cats = defaultdict(int)
    for line in losses:
        if "length=" in line: cats["length dropped"] += 1
        elif "precision=" in line or "scale=" in line: cats["precision/scale dropped"] += 1
        elif "collapsed to pack `double`" in line: cats["int/long → double"] += 1
        elif "sharingModel and enableHierarchySharing intentionally not extracted" in line: cats["sharingModel/enableHierarchy declined (L0 fallback)"] += 1
        elif "recordTypeInfos" in line: cats["recordTypeInfos not embedded"] += 1
        elif "no authoritative childRelationshipName" in line: cats["childRelationshipName missing (parent describe absent or SF null)"] += 1
        elif "polymorphic FK has divergent childRelationshipName" in line: cats["polymorphic childRelationshipName divergent across parents"] += 1
        elif "no pack-side mapping" in line: cats["unmapped SF type"] += 1
        else: cats["other"] += 1
    for k, v in sorted(cats.items(), key=lambda x: -x[1]):
        out.append(f"- {k}: {v}")
    OUT_LOSS_REPORT.write_text("\n".join(out) + "\n")

    print(f"objects:      {len(objs_out)}")
    print(f"components:   {len(components)}")
    print(f"out objects:  {OUT_OBJECTS}")
    print(f"out manifest: {OUT_MANIFEST_SNIPPET}")
    print(f"out losses:   {OUT_LOSS_REPORT}")
    return 0


if __name__ == "__main__":
    sys.exit(main())