unik3d/utils/validation.py

import json
import os
from collections import defaultdict
from typing import Any, Dict, List, Optional

import numpy as np
import torch
import torch.distributed as dist
import torch.utils.data.distributed
import wandb
from PIL import Image
from torch.nn import functional as F
from torch.utils.data import DataLoader
from tqdm import tqdm

from unik3d.utils.distributed import barrier, get_world_size, is_main_process
from unik3d.utils.misc import remove_leading_dim, remove_padding, ssi_helper
from unik3d.utils.visualization import colorize, image_grid


def stack_mixedshape_numpy(tensor_list, dim=0):
    max_rows = max(tensor.shape[0] for tensor in tensor_list)
    max_columns = max(tensor.shape[1] for tensor in tensor_list)

    padded_tensors = []
    for tensor in tensor_list:
        rows, columns, *_ = tensor.shape
        pad_rows = max_rows - rows
        pad_columns = max_columns - columns

        padded_tensor = np.pad(
            tensor, ((0, pad_rows), (0, pad_columns), (0, 0)), mode="constant"
        )
        padded_tensors.append(padded_tensor)

    return np.stack(padded_tensors, axis=dim)


def original_image(batch):
    paddings = [
        torch.tensor(pads)
        for img_meta in batch["img_metas"]
        for pads in img_meta.get("paddings", [[0] * 4])
    ]
    paddings = torch.stack(paddings).to(batch["data"]["image"].device)[
        ..., [0, 2, 1, 3]
    ]  # lrtb

    T, _, H, W = batch["data"]["depth"].shape
    batch["data"]["image"] = F.interpolate(
        batch["data"]["image"],
        (H + paddings[0][2] + paddings[0][3], W + paddings[0][1] + paddings[0][2]),
        mode="bilinear",
        align_corners=False,
        antialias=True,
    )
    batch["data"]["image"] = remove_padding(
        batch["data"]["image"], paddings.repeat(T, 1)
    )
    return batch


def original_image_inv(batch, preds=None):
    paddings = [
        torch.tensor(pads)
        for img_meta in batch["img_metas"]
        for pads in img_meta.get("padding_size", [[0] * 4])
    ]
    T, _, H, W = batch["data"]["depth"].shape
    batch["data"]["image"] = remove_padding(batch["data"]["image"], paddings * T)
    batch["data"]["image"] = F.interpolate(
        batch["data"]["image"],
        (H, W),
        mode="bilinear",
        align_corners=False,
        antialias=True,
    )

    if preds is not None:
        for key in ["depth"]:
            if key in preds:
                preds[key] = remove_padding(preds[key], paddings * T)
                preds[key] = F.interpolate(
                    preds[key],
                    (H, W),
                    mode="bilinear",
                    align_corners=False,
                    antialias=True,
                )

    return batch, preds


def aggregate_metrics(metrics_all, exclude_fn=lambda name: False):
    aggregate_name = "".join(
        [name_ds[:3] for name_ds in metrics_all.keys() if not exclude_fn(name_ds)]
    )
    metrics_aggregate = defaultdict(list)
    for name_ds, metrics in metrics_all.items():
        if exclude_fn(name_ds):
            continue
        for metrics_name, metrics_value in metrics.items():
            metrics_aggregate[metrics_name].append(metrics_value)
    return {
        **{aggregate_name: {k: sum(v) / len(v) for k, v in metrics_aggregate.items()}},
        **metrics_all,
    }


GROUPS = {
    "SFoV": ["KITTI", "NYUv2Depth", "DiodeIndoor", "ETH3D", "IBims"],
    "SFoVDi": ["DiodeIndoor_F", "ETH3D_F", "IBims_F"],
    "LFoV": ["ADT", "KITTI360", "ScanNetpp_F"],
}


def aggregate_metrics_camera(metrics_all):
    available_groups = {
        k: v for k, v in GROUPS.items() if any([name in metrics_all for name in v])
    }
    for group_name, group_datasets in available_groups.items():
        metrics_aggregate = defaultdict(list)
        for dataset_name in group_datasets:
            if dataset_name not in metrics_all:
                print(
                    f"Dataset {dataset_name} not used for aggregation of {group_name}"
                )
                continue
            for metrics_name, metrics_value in metrics_all[dataset_name].items():
                metrics_aggregate[metrics_name].append(metrics_value)
        metrics_all[group_name] = {
            k: sum(v) / len(v) for k, v in metrics_aggregate.items()
        }
    return metrics_all


def log_metrics(metrics_all, step):
    for name_ds, metrics in metrics_all.items():
        for metrics_name, metrics_value in metrics.items():
            try:
                wandb.log(
                    {f"Metrics/{name_ds}/{metrics_name}": metrics_value}, step=step
                )
            except:
                print(f"Metrics/{name_ds}/{metrics_name} {round(metrics_value, 4)}")


def log_artifacts(artifacts_all, step, run_id):
    for ds_name, artifacts in artifacts_all.items():
        rgbs, gts = artifacts["rgbs"], artifacts["gts"]
        logging_imgs = [
            *rgbs,
            *gts,
            *[
                x
                for k, v in artifacts.items()
                if ("rgbs" not in k and "gts" not in k)
                for x in v
            ],
        ]
        artifacts_grid = image_grid(logging_imgs, len(artifacts), len(rgbs))
        try:
            wandb.log({f"{ds_name}_test": [wandb.Image(artifacts_grid)]}, step=step)
        except:
            print(f"Error while saving artifacts at step {step}")


def show(vals, dataset, ssi_depth=False):
    output_artifacts, additionals = {}, {}
    predictions, gts, errors, images = [], [], [], []
    for v in vals:
        image = v["image"][0].unsqueeze(0)
        gt = v["depth"][0].unsqueeze(0)
        prediction = v["depth_pred"][0].unsqueeze(0)
        # Downsample for memory and viz
        # if any([x in dataset.__class__.__name__ for x in ["DDAD", "Argoverse", "Waymo", "DrivingStereo"]]):
        #     gt = F.interpolate(gt, scale_factor=0.5, mode="nearest-exact")
        #     # Dilate for a better visualization
        #     gt[gt < 1e-4] = dilate(gt)[gt < 1e-4]
        H, W = gt.shape[-2:]
        aspect_ratio = H / W
        new_W = int((300_000 / aspect_ratio) ** 0.5)
        new_H = int(aspect_ratio * new_W)
        gt = F.interpolate(gt, (new_H, new_W), mode="nearest-exact")

        # Format predictions and errors for every metrics used
        prediction = F.interpolate(
            prediction,
            gt.shape[-2:],
            mode="bilinear",
            align_corners=False,
            antialias=True,
        )
        error = torch.zeros_like(prediction)
        error[gt > dataset.min_depth] = (
            4
            * dataset.max_depth
            * torch.abs(gt - prediction)[gt > dataset.min_depth]
            / gt[gt > dataset.min_depth]
        )
        if ssi_depth:
            scale, shift = ssi_helper(gt[gt > 0], prediction[gt > 0])
            prediction = (prediction * scale + shift).clip(0.0, dataset.max_depth)
        prediction = colorize(
            prediction.squeeze().cpu().detach().numpy(),
            vmin=dataset.min_depth,
            vmax=dataset.max_depth,
            cmap="magma_r",
        )
        error = error.clip(0.0, dataset.max_depth).cpu().detach().numpy()
        error = colorize(error.squeeze(), vmin=0.001, vmax=1.0, cmap="coolwarm")
        errors.append(error)
        predictions.append(prediction)

        image = F.interpolate(
            image, gt.shape[-2:], mode="bilinear", align_corners=False, antialias=True
        )
        image = image.cpu().detach() * dataset.normalization_stats["std"].view(
            1, -1, 1, 1
        ) + dataset.normalization_stats["mean"].view(1, -1, 1, 1)
        image = (
            (255 * image)
            .clip(0.0, 255.0)
            .to(torch.uint8)
            .permute(0, 2, 3, 1)
            .numpy()
            .squeeze()
        )
        gt = gt.clip(0.0, dataset.max_depth).cpu().detach().numpy()
        gt = colorize(
            gt.squeeze(), vmin=dataset.min_depth, vmax=dataset.max_depth, cmap="magma_r"
        )
        gts.append(gt)
        images.append(image)

        for name, additional in v.get("infos", {}).items():
            if name not in additionals:
                additionals[name] = []
            if additional[0].shape[0] == 3:
                val = (
                    (127.5 * (additional[0] + 1))
                    .clip(0, 255)
                    .to(torch.uint8)
                    .cpu()
                    .detach()
                    .permute(1, 2, 0)
                    .numpy()
                )
            else:
                val = colorize(
                    additional[0].cpu().detach().squeeze().numpy(),
                    0.0,
                    dataset.max_depth,
                )
            additionals[name].append(val)

    output_artifacts.update(
        {
            f"predictions": stack_mixedshape_numpy(predictions),
            f"errors": stack_mixedshape_numpy(errors),
            "rgbs": stack_mixedshape_numpy(images),
            "gts": stack_mixedshape_numpy(gts),
            **{k: stack_mixedshape_numpy(v) for k, v in additionals.items()},
        }
    )
    return output_artifacts


METRIC_B = "F1"
INVERT = True
SSI_VISUALIZATION = True


def validate(
    model,
    test_loaders: Dict[str, DataLoader],
    step,
    run_id,
    context,
    idxs=(1, 100, 150, 1000),
):

    metrics_all, predictions_select = {}, {}
    world_size = get_world_size()
    for name_ds, test_loader in test_loaders.items():
        idxs = [idx % len(test_loader.dataset) for idx in idxs]
        ds_show = []
        for i, batch in enumerate(test_loader):
            with context:
                batch["data"] = {
                    k: v.to(model.device) for k, v in batch["data"].items()
                }
                preds = model(batch["data"], batch["img_metas"])

            if batch["data"]["image"].ndim == 5:
                batch["data"] = remove_leading_dim(batch["data"])
            if preds["depth"].ndim == 5:
                preds = remove_leading_dim(preds)
            batch = original_image(batch)
            test_loader.dataset.accumulate_metrics(
                inputs=batch["data"],
                preds=preds,
                keyframe_idx=batch["img_metas"][0].get("keyframe_idx"),
            )

            # for prediction images logging
            if i * world_size in idxs:
                ii = (len(preds["depth"]) + 1) // 2 - 1
                slice_ = slice(ii, ii + 1)
                batch["data"] = {k: v[slice_] for k, v in batch["data"].items()}
                preds["depth"] = preds["depth"][slice_]
                ds_show.append({**batch["data"], **{"depth_pred": preds["depth"]}})

        barrier()

        metrics_all[name_ds] = test_loader.dataset.get_evaluation()
        predictions_select[name_ds] = show(
            ds_show, test_loader.dataset, ssi_depth=SSI_VISUALIZATION
        )

    barrier()
    if is_main_process():
        log_artifacts(artifacts_all=predictions_select, step=step, run_id=run_id)
        metrics_all = aggregate_metrics(
            metrics_all, exclude_fn=lambda name: "mono" in name
        )
        metrics_all = aggregate_metrics_camera(metrics_all)
        log_metrics(metrics_all=metrics_all, step=step)
    return metrics_all