third_party/dust3r/croco/stereoflow/engine.py

# Copyright (C) 2022-present Naver Corporation. All rights reserved.
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).

# --------------------------------------------------------
# Main function for training one epoch or testing
# --------------------------------------------------------

import math
import sys
from typing import Iterable

import numpy as np
import torch
import torchvision
from utils import misc as misc


def split_prediction_conf(predictions, with_conf=False):
    if not with_conf:
        return predictions, None
    conf = predictions[:, -1:, :, :]
    predictions = predictions[:, :-1, :, :]
    return predictions, conf


def train_one_epoch(
    model: torch.nn.Module,
    criterion: torch.nn.Module,
    metrics: torch.nn.Module,
    data_loader: Iterable,
    optimizer: torch.optim.Optimizer,
    device: torch.device,
    epoch: int,
    loss_scaler,
    log_writer=None,
    print_freq=20,
    args=None,
):
    model.train(True)
    metric_logger = misc.MetricLogger(delimiter="  ")
    metric_logger.add_meter("lr", misc.SmoothedValue(window_size=1, fmt="{value:.6f}"))
    header = "Epoch: [{}]".format(epoch)

    accum_iter = args.accum_iter

    optimizer.zero_grad()

    details = {}

    if log_writer is not None:
        print("log_dir: {}".format(log_writer.log_dir))

    if args.img_per_epoch:
        iter_per_epoch = args.img_per_epoch // args.batch_size + int(
            args.img_per_epoch % args.batch_size > 0
        )
        assert (
            len(data_loader) >= iter_per_epoch
        ), "Dataset is too small for so many iterations"
        len_data_loader = iter_per_epoch
    else:
        len_data_loader, iter_per_epoch = len(data_loader), None

    for data_iter_step, (image1, image2, gt, pairname) in enumerate(
        metric_logger.log_every(
            data_loader, print_freq, header, max_iter=iter_per_epoch
        )
    ):
        image1 = image1.to(device, non_blocking=True)
        image2 = image2.to(device, non_blocking=True)
        gt = gt.to(device, non_blocking=True)

        # we use a per iteration (instead of per epoch) lr scheduler
        if data_iter_step % accum_iter == 0:
            misc.adjust_learning_rate(
                optimizer, data_iter_step / len_data_loader + epoch, args
            )

        with torch.cuda.amp.autocast(enabled=bool(args.amp)):
            prediction = model(image1, image2)
            prediction, conf = split_prediction_conf(prediction, criterion.with_conf)
            batch_metrics = metrics(prediction.detach(), gt)
            loss = (
                criterion(prediction, gt)
                if conf is None
                else criterion(prediction, gt, conf)
            )

        loss_value = loss.item()
        if not math.isfinite(loss_value):
            print("Loss is {}, stopping training".format(loss_value))
            sys.exit(1)

        loss /= accum_iter
        loss_scaler(
            loss,
            optimizer,
            parameters=model.parameters(),
            update_grad=(data_iter_step + 1) % accum_iter == 0,
        )
        if (data_iter_step + 1) % accum_iter == 0:
            optimizer.zero_grad()

        torch.cuda.synchronize()

        metric_logger.update(loss=loss_value)
        for k, v in batch_metrics.items():
            metric_logger.update(**{k: v.item()})
        lr = optimizer.param_groups[0]["lr"]
        metric_logger.update(lr=lr)

        # if args.dsitributed: loss_value_reduce = misc.all_reduce_mean(loss_value)
        time_to_log = (data_iter_step + 1) % (
            args.tboard_log_step * accum_iter
        ) == 0 or data_iter_step == len_data_loader - 1
        loss_value_reduce = misc.all_reduce_mean(loss_value)
        if log_writer is not None and time_to_log:
            epoch_1000x = int((data_iter_step / len_data_loader + epoch) * 1000)
            # We use epoch_1000x as the x-axis in tensorboard. This calibrates different curves when batch size changes.
            log_writer.add_scalar("train/loss", loss_value_reduce, epoch_1000x)
            log_writer.add_scalar("lr", lr, epoch_1000x)
            for k, v in batch_metrics.items():
                log_writer.add_scalar("train/" + k, v.item(), epoch_1000x)

    # gather the stats from all processes
    # if args.distributed: metric_logger.synchronize_between_processes()
    print("Averaged stats:", metric_logger)
    return {k: meter.global_avg for k, meter in metric_logger.meters.items()}


@torch.no_grad()
def validate_one_epoch(
    model: torch.nn.Module,
    criterion: torch.nn.Module,
    metrics: torch.nn.Module,
    data_loaders: list[Iterable],
    device: torch.device,
    epoch: int,
    log_writer=None,
    args=None,
):
    model.eval()
    metric_loggers = []
    header = "Epoch: [{}]".format(epoch)
    print_freq = 20

    conf_mode = args.tile_conf_mode
    crop = args.crop

    if log_writer is not None:
        print("log_dir: {}".format(log_writer.log_dir))

    results = {}
    dnames = []
    image1, image2, gt, prediction = None, None, None, None
    for didx, data_loader in enumerate(data_loaders):
        dname = str(data_loader.dataset)
        dnames.append(dname)
        metric_loggers.append(misc.MetricLogger(delimiter="  "))
        for data_iter_step, (image1, image2, gt, pairname) in enumerate(
            metric_loggers[didx].log_every(data_loader, print_freq, header)
        ):
            image1 = image1.to(device, non_blocking=True)
            image2 = image2.to(device, non_blocking=True)
            gt = gt.to(device, non_blocking=True)
            if dname.startswith("Spring"):
                assert (
                    gt.size(2) == image1.size(2) * 2
                    and gt.size(3) == image1.size(3) * 2
                )
                gt = (
                    gt[:, :, 0::2, 0::2]
                    + gt[:, :, 0::2, 1::2]
                    + gt[:, :, 1::2, 0::2]
                    + gt[:, :, 1::2, 1::2]
                ) / 4.0  # we approximate the gt based on the 2x upsampled ones

            with torch.inference_mode():
                prediction, tiled_loss, c = tiled_pred(
                    model,
                    criterion,
                    image1,
                    image2,
                    gt,
                    conf_mode=conf_mode,
                    overlap=args.val_overlap,
                    crop=crop,
                    with_conf=criterion.with_conf,
                )
                batch_metrics = metrics(prediction.detach(), gt)
                loss = (
                    criterion(prediction.detach(), gt)
                    if not criterion.with_conf
                    else criterion(prediction.detach(), gt, c)
                )
                loss_value = loss.item()
                metric_loggers[didx].update(loss_tiled=tiled_loss.item())
                metric_loggers[didx].update(**{f"loss": loss_value})
                for k, v in batch_metrics.items():
                    metric_loggers[didx].update(**{dname + "_" + k: v.item()})

    results = {
        k: meter.global_avg for ml in metric_loggers for k, meter in ml.meters.items()
    }
    if len(dnames) > 1:
        for k in batch_metrics.keys():
            results["AVG_" + k] = sum(
                results[dname + "_" + k] for dname in dnames
            ) / len(dnames)

    if log_writer is not None:
        epoch_1000x = int((1 + epoch) * 1000)
        for k, v in results.items():
            log_writer.add_scalar("val/" + k, v, epoch_1000x)

    print("Averaged stats:", results)
    return results


import torch.nn.functional as F


def _resize_img(img, new_size):
    return F.interpolate(img, size=new_size, mode="bicubic", align_corners=False)


def _resize_stereo_or_flow(data, new_size):
    assert data.ndim == 4
    assert data.size(1) in [1, 2]
    scale_x = new_size[1] / float(data.size(3))
    out = F.interpolate(data, size=new_size, mode="bicubic", align_corners=False)
    out[:, 0, :, :] *= scale_x
    if out.size(1) == 2:
        scale_y = new_size[0] / float(data.size(2))
        out[:, 1, :, :] *= scale_y
        print(scale_x, new_size, data.shape)
    return out


@torch.no_grad()
def tiled_pred(
    model,
    criterion,
    img1,
    img2,
    gt,
    overlap=0.5,
    bad_crop_thr=0.05,
    downscale=False,
    crop=512,
    ret="loss",
    conf_mode="conf_expsigmoid_10_5",
    with_conf=False,
    return_time=False,
):
    # for each image, we are going to run inference on many overlapping patches
    # then, all predictions will be weighted-averaged
    if gt is not None:
        B, C, H, W = gt.shape
    else:
        B, _, H, W = img1.shape
        C = model.head.num_channels - int(with_conf)
    win_height, win_width = crop[0], crop[1]

    # upscale to be larger than the crop
    do_change_scale = H < win_height or W < win_width
    if do_change_scale:
        upscale_factor = max(win_width / W, win_height / W)
        original_size = (H, W)
        new_size = (round(H * upscale_factor), round(W * upscale_factor))
        img1 = _resize_img(img1, new_size)
        img2 = _resize_img(img2, new_size)
        # resize gt just for the computation of tiled losses
        if gt is not None:
            gt = _resize_stereo_or_flow(gt, new_size)
        H, W = img1.shape[2:4]

    if conf_mode.startswith("conf_expsigmoid_"):  # conf_expsigmoid_30_10
        beta, betasigmoid = map(float, conf_mode[len("conf_expsigmoid_") :].split("_"))
    elif conf_mode.startswith("conf_expbeta"):  # conf_expbeta3
        beta = float(conf_mode[len("conf_expbeta") :])
    else:
        raise NotImplementedError(f"conf_mode {conf_mode} is not implemented")

    def crop_generator():
        for sy in _overlapping(H, win_height, overlap):
            for sx in _overlapping(W, win_width, overlap):
                yield sy, sx, sy, sx, True

    # keep track of weighted sum of prediction*weights and weights
    accu_pred = img1.new_zeros(
        (B, C, H, W)
    )  # accumulate the weighted sum of predictions
    accu_conf = img1.new_zeros((B, H, W)) + 1e-16  # accumulate the weights
    accu_c = img1.new_zeros(
        (B, H, W)
    )  # accumulate the weighted sum of confidences ; not so useful except for computing some losses

    tiled_losses = []

    if return_time:
        start = torch.cuda.Event(enable_timing=True)
        end = torch.cuda.Event(enable_timing=True)
        start.record()

    for sy1, sx1, sy2, sx2, aligned in crop_generator():
        # compute optical flow there
        pred = model(_crop(img1, sy1, sx1), _crop(img2, sy2, sx2))
        pred, predconf = split_prediction_conf(pred, with_conf=with_conf)

        if gt is not None:
            gtcrop = _crop(gt, sy1, sx1)
        if criterion is not None and gt is not None:
            tiled_losses.append(
                criterion(pred, gtcrop).item()
                if predconf is None
                else criterion(pred, gtcrop, predconf).item()
            )

        if conf_mode.startswith("conf_expsigmoid_"):
            conf = torch.exp(
                -beta * 2 * (torch.sigmoid(predconf / betasigmoid) - 0.5)
            ).view(B, win_height, win_width)
        elif conf_mode.startswith("conf_expbeta"):
            conf = torch.exp(-beta * predconf).view(B, win_height, win_width)
        else:
            raise NotImplementedError

        accu_pred[..., sy1, sx1] += pred * conf[:, None, :, :]
        accu_conf[..., sy1, sx1] += conf
        accu_c[..., sy1, sx1] += predconf.view(B, win_height, win_width) * conf

    pred = accu_pred / accu_conf[:, None, :, :]
    c = accu_c / accu_conf
    assert not torch.any(torch.isnan(pred))

    if return_time:
        end.record()
        torch.cuda.synchronize()
        time = start.elapsed_time(end) / 1000.0  # this was in milliseconds

    if do_change_scale:
        pred = _resize_stereo_or_flow(pred, original_size)

    if return_time:
        return pred, torch.mean(torch.tensor(tiled_losses)), c, time
    return pred, torch.mean(torch.tensor(tiled_losses)), c


def _overlapping(total, window, overlap=0.5):
    assert total >= window and 0 <= overlap < 1, (total, window, overlap)
    num_windows = 1 + int(np.ceil((total - window) / ((1 - overlap) * window)))
    offsets = np.linspace(0, total - window, num_windows).round().astype(int)
    yield from (slice(x, x + window) for x in offsets)


def _crop(img, sy, sx):
    B, THREE, H, W = img.shape
    if 0 <= sy.start and sy.stop <= H and 0 <= sx.start and sx.stop <= W:
        return img[:, :, sy, sx]
    l, r = max(0, -sx.start), max(0, sx.stop - W)
    t, b = max(0, -sy.start), max(0, sy.stop - H)
    img = torch.nn.functional.pad(img, (l, r, t, b), mode="constant")
    return img[:, :, slice(sy.start + t, sy.stop + t), slice(sx.start + l, sx.stop + l)]