Remove legacy files and configurations in preparation for a new project structure

- Deleted DVC-related files including .dvcignore, dvc.lock, dvc.yaml, and model files.
- Removed training and evaluation scripts, dataset processing scripts, and associated parameters.
- Cleaned up the project by deleting unused directories and files, including logs and requirements for Heroku deployment.
- Updated the app to streamline the inference process and improve demo mode functionality.

This commit sets the foundation for a simplified deployment and development workflow.

.dvc/.gitignore

@@ -1,3 +0,0 @@
-/config.local
-/tmp
-/cache

.dvc/config

@@ -1,7 +0,0 @@
-[core]
-    analytics = false
-    remote = origin
-['remote "origin"']
-    url = https://dagshub.com/OperationSavta/SavtaDepth.dvc
-['remote "upstream"']
-    url = https://dagshub.com/OperationSavta/SavtaDepth.dvc

.dvc/plots/confusion.json

@@ -1,30 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "mark": "rect",
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "nominal",
-            "sort": "ascending",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "nominal",
-            "sort": "ascending",
-            "title": "<DVC_METRIC_Y_LABEL>"
-        },
-        "color": {
-            "aggregate": "count",
-            "type": "quantitative"
-        },
-        "facet": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    }
-}

.dvc/plots/default.json

@@ -1,29 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "mark": {
-        "type": "line"
-    },
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_Y_LABEL>",
-            "scale": {
-                "zero": false
-            }
-        },
-        "color": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    }
-}

.dvc/plots/scatter.json

@@ -1,27 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "mark": "point",
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_Y_LABEL>",
-            "scale": {
-                "zero": false
-            }
-        },
-        "color": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    }
-}

.dvc/plots/smooth.json

@@ -1,39 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "mark": {
-        "type": "line"
-    },
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_Y_LABEL>",
-            "scale": {
-                "zero": false
-            }
-        },
-        "color": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    },
-    "transform": [
-        {
-            "loess": "<DVC_METRIC_Y>",
-            "on": "<DVC_METRIC_X>",
-            "groupby": [
-                "rev"
-            ],
-            "bandwidth": 0.3
-        }
-    ]
-}

.dvcignore

@@ -1,3 +0,0 @@
-# Add patterns of files dvc should ignore, which could improve
-# the performance. Learn more at
-# https://dvc.org/doc/user-guide/dvcignore

LICENSE

@@ -1,8 +0,0 @@
-MIT License
-Copyright (c) 2021 DAGsHub
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -1,37 +0,0 @@
-#################################################################################
-# GLOBALS                                                                       #
-#################################################################################
-
-PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-PROJECT_NAME = savta_depth
-PYTHON_INTERPRETER = python3
-
-ifeq (,$(shell which conda))
-HAS_CONDA=False
-else
-HAS_CONDA=True
-endif
-
-#################################################################################
-# COMMANDS                                                                      #
-#################################################################################
-
-env:
-ifeq (True,$(HAS_CONDA))
-	@echo ">>> Detected conda, creating conda environment."
-	conda create -y --name $(PROJECT_NAME) python=3.7.6 
-	@echo ">>> New conda env created. Activate with:\nconda activate $(PROJECT_NAME)"
-else
-	@echo ">>> No conda detected, creating venv environment."
-	$(PYTHON_INTERPRETER) -m venv env
-	@echo ">>> New virtual env created. Activate with:\nsource env/bin/activate ."
-endif
-
-load_requirements:
-	@echo ">>> Installing requirements. Make sure your virtual environment is activated."
-	$(PYTHON_INTERPRETER) -m pip install -U pip setuptools wheel
-	$(PYTHON_INTERPRETER) -m pip install -r requirements.txt
-
-save_requirements:
-	@echo ">>> Saving requirements."
-	pip list --format=freeze > requirements.txt

README_HF.md

@@ -1,53 +0,0 @@
-# SavtaDepth - Hugging Face Spaces
-
-This is a Gradio demo for monocular depth estimation.
-
-## 🚀 Quick Start
-
-The Space is configured to run without a model by default. To enable full depth estimation functionality:
-
-### Option 1: Upload via Hugging Face Space Settings
-1. Go to your Space settings
-2. Upload your trained `model.pth` file to the `models/` directory
-3. Restart the Space
-
-### Option 2: For Local Development
-1. Clone this repository
-2. Place your trained model at `models/model.pth`
-3. Run locally: `python app/app_savta.py`
-
-## 📋 Requirements
-
-The model should be:
-- A PyTorch model file (.pth extension)
-- Compatible with UNet architecture with ResNet34 backbone
-- Trained for depth estimation
-
-## 🎯 Demo Mode
-
-When no model is present, the app will:
-- Show a warning message in orange
-- Convert uploaded images to grayscale instead of depth prediction
-- Allow you to test the UI functionality
-
-## 🛠️ Model Training
-
-To train your own model, refer to the original project repository at:
-https://dagshub.com/OperationSavta/SavtaDepth
-
-## 📊 Model Architecture
-
-- **Backbone**: ResNet34
-- **Architecture**: UNet
-- **Input**: RGB images (any size)
-- **Output**: Depth maps (single channel grayscale)
-
-## ⚡ Performance
-
-- **Inference Time**: ~100ms per image (depending on hardware)
-- **Memory Usage**: ~500MB VRAM
-- **Supported Formats**: JPG, PNG
-
----
-
-*Built with [Gradio](https://gradio.app/) and [FastAI](https://www.fast.ai/)*

app/app_savta.py

@@ -1,157 +1,37 @@
-import os, sys, tempfile, subprocess
 from pathlib import Path
 
-import torch
-from fastai.vision.all import *
 import gradio as gr
+from fastai.vision.all import *
 
-#######################
-# Hugging Face flags  #
-#######################
-
-HF_TOKEN = os.getenv("HF_TOKEN")
-
-try:
-    from gradio.flagging import HuggingFaceDatasetSaver  # type: ignore
-
-    hf_writer: gr.FlaggingCallback | None = HuggingFaceDatasetSaver(
-        repo_id="savtadepth-flags-V2", token=HF_TOKEN
-    )
-    allow_flagging: str | bool = "manual"
-except (ImportError, AttributeError):
-    hf_writer = None
-    allow_flagging = "never"  # hide flag button if callback unavailable
-
-############
-#   DVC    #
-############
-
-DAGSHUB_USER = os.getenv("DAGSHUB_USER")
-DAGSHUB_TOKEN = os.getenv("DAGSHUB_USER_TOKEN")
-
-if not DAGSHUB_USER or not DAGSHUB_TOKEN:
-    sys.exit("❌ Please set DAGSHUB_USER and DAGSHUB_USER_TOKEN environment variables.")
-
-REPO = "kingabzpro/SavtaDepth"
-PROD_MODEL_PATH = Path("src/models/model.pth")
-TRAIN_DIR       = Path("src/data/processed/train/bathroom")
-TEST_DIR        = Path("src/data/processed/test/bathroom")
-
-def _run(cmd: list[str]):
-    return subprocess.run(cmd, text=True, capture_output=True)
-
-def dagshub_download(remote: str, local: Path):
-    local.parent.mkdir(parents=True, exist_ok=True)
-    cmd = ["dagshub", "download", REPO, remote, str(local)]
-    res = _run(cmd)
-    if res.returncode != 0:
-        sys.exit(f"❌ dagshub download failed\nRemote: {remote}\nSTDERR:\n{res.stderr}\nSTDOUT:\n{res.stdout}")
-    print(f"✅ Downloaded {remote} → {local}")
-
-# 1) Model file
-if not PROD_MODEL_PATH.exists():
-    dagshub_download("src/models/model.pth", PROD_MODEL_PATH)
-
-# 2) Train/Test directories (recursive)
-if not TRAIN_DIR.exists():
-    dagshub_download("src/data/processed/train/bathroom", TRAIN_DIR)
-if not TEST_DIR.exists():
-    dagshub_download("src/data/processed/test/bathroom", TEST_DIR)
-
-
-#######################
-#   Data & Learner    #
-#######################
-
-class ImageImageDataLoaders(DataLoaders):
-    """Create DataLoaders for image→image tasks."""
-
-    @classmethod
-    @delegates(DataLoaders.from_dblock)
-    def from_label_func(
-        cls,
-        path: Path,
-        filenames,
-        label_func,
-        valid_pct: float = 0.2,
-        seed: int | None = None,
-        item_transforms=None,
-        batch_transforms=None,
-        **kwargs,
-    ):
-        dblock = DataBlock(
-            blocks=(ImageBlock(cls=PILImage), ImageBlock(cls=PILImageBW)),
-            get_y=label_func,
-            splitter=RandomSplitter(valid_pct, seed=seed),
-            item_tfms=item_transforms,
-            batch_tfms=batch_transforms,
-        )
-        return cls.from_dblock(dblock, filenames, path=path, **kwargs)
-
-
-def get_y_fn(x: Path) -> Path:
-    return Path(str(x).replace(".jpg", "_depth.png"))
-
-
-def create_data(data_path: Path):
-    fnames = get_files(data_path / "train", extensions=".jpg")
-    return ImageImageDataLoaders.from_label_func(
-        data_path / "train",
-        seed=42,
-        bs=4,
-        num_workers=0,
-        filenames=fnames,
-        label_func=get_y_fn,
-    )
+# Model setup
+MODEL_PATH = Path(__file__).parent.parent / "models" / "model.pth"
 
+if not MODEL_PATH.exists():
+    raise FileNotFoundError(f"Model not found at {MODEL_PATH}")
 
-data = create_data(Path("src/data/processed"))
-learner = unet_learner(
-    data,
-    resnet34,
-    metrics=rmse,
-    wd=1e-2,
-    n_out=3,
-    loss_func=MSELossFlat(),
-    path="src/",
-)
-learner.load("model")
+learner = load_learner(MODEL_PATH)
 
-#####################
-#  Inference Logic  #
-#####################
 
+# Inference function
 def predict_depth(input_img: PILImage) -> PILImageBW:
     depth, *_ = learner.predict(input_img)
     return PILImageBW.create(depth).convert("L")
 
-#####################
-#    Gradio UI      #
-#####################
 
-title = "📷 SavtaDepth WebApp"
+# Gradio UI
+title = "📷 SavtaDepth WebApp"
 
-description_md = (
-    """
+description_md = """
     <p style="text-align:center;font-size:1.05rem;max-width:760px;margin:auto;">
         Upload an RGB image on the left and get a grayscale depth map on the right.
     </p>
     """
-)
 
-footer_html = (
-    """
-    <p style='text-align:center;font-size:0.9rem;'>
-        <a href='https://dagshub.com/OperationSavta/SavtaDepth' target='_blank'>Project on DAGsHub</a> •
-        <a href='https://colab.research.google.com/drive/1XU4DgQ217_hUMU1dllppeQNw3pTRlHy1?usp=sharing' target='_blank'>Google Colab Demo</a>
-    </p>
-    """
-)
-
-examples = [["examples/00008.jpg"], ["examples/00045.jpg"]]
+examples_dir = Path(__file__).parent.parent / "examples"
+examples = [[str(examples_dir / "00008.jpg")], [str(examples_dir / "00045.jpg")]]
 
-input_component = gr.Image(width=640, height=480, label="Input RGB")
-output_component = gr.Image(label="Predicted Depth", image_mode="L")
+input_component = gr.Image(width=640, height=480, label="Input RGB")
+output_component = gr.Image(label="Predicted Depth", image_mode="L")
 
 with gr.Blocks(title=title, theme=gr.themes.Soft()) as demo:
     gr.Markdown(f"<center><h1>{title}</h1></center>")
@@ -161,14 +41,9 @@ with gr.Blocks(title=title, theme=gr.themes.Soft()) as demo:
         fn=predict_depth,
         inputs=input_component,
         outputs=output_component,
-        allow_flagging=allow_flagging,
-        flagging_options=["incorrect", "worst", "ambiguous"],
-        flagging_callback=hf_writer,
         examples=examples,
         cache_examples=False,
     )
 
-    gr.HTML(footer_html)
-
 if __name__ == "__main__":
     demo.queue().launch()

dvc.lock

@@ -1,75 +0,0 @@
-process_data:
-  cmd: python3 src/code/make_dataset.py src/data/raw/nyu_depth_v2_labeled.mat src/data/raw/splits.mat
-    src/data/processed
-  deps:
-  - path: src/code/make_dataset.py
-    md5: e069c7323c9be16baedd8f988375e145
-    size: 5256
-  - path: src/data/raw/nyu_depth_v2_labeled.mat
-    md5: 520609c519fba3ba5ac58c8fefcc3530
-    size: 2972037809
-  - path: src/data/raw/splits.mat
-    md5: 08e3c3aea27130ac7c01ffd739a4535f
-    size: 2626
-  outs:
-  - path: src/data/processed/
-    md5: 9a1f43f46e8b1c387532e994e721d57d.dir
-    size: 197717291
-    nfiles: 2898
-train:
-  cmd: python3 src/code/training.py src/data/processed/train
-  deps:
-  - path: src/code/custom_data_loading.py
-    md5: c94ea029ed76ca94bb1ad4c1655e5e68
-    size: 1916
-  - path: src/code/params.yml
-    md5: 88b982495a09b6d9355903e31e0b3b3f
-    size: 218
-  - path: src/code/training.py
-    md5: e3dff7f4b59e4ebf818d7631d3e6803a
-    size: 1683
-  - path: src/data/processed/train
-    md5: 9956d748dcadc3abadd1ff966a6e2b92.dir
-    size: 109120921
-    nfiles: 1590
-  outs:
-  - path: logs/train_metrics.csv
-    md5: 15b14c6b8a3c310a7149b3f1bac5d86f
-    size: 5920790
-  - path: logs/train_params.yml
-    md5: 4d148d75cab3dbaa91ec5fccb3382541
-    size: 887
-  - path: src/models/
-    md5: 8586da76f372efa83d832a9d0e664817.dir
-    size: 494927324
-    nfiles: 1
-eval:
-  cmd: python3 src/code/eval.py src/data/processed/test
-  deps:
-  - path: src/code/custom_data_loading.py
-    md5: c94ea029ed76ca94bb1ad4c1655e5e68
-    size: 1916
-  - path: src/code/eval.py
-    md5: 9ea6a6624fa14f15b4d51f9139395663
-    size: 1893
-  - path: src/code/eval_metric_calculation.py
-    md5: 2fc866e1107042a996087d5716d44bf0
-    size: 2999
-  - path: src/code/params.yml
-    md5: 88b982495a09b6d9355903e31e0b3b3f
-    size: 218
-  - path: src/data/processed/test
-    md5: bcccd66f3f561b53ba97c89a558c08a0.dir
-    size: 88596370
-    nfiles: 1308
-  - path: src/models/model.pth
-    md5: f421fb113498c7186fb734928484e013
-    size: 494927324
-  outs:
-  - path: logs/test_metrics.csv
-    md5: ee70c01208cdd018a567debd0abb1643
-    size: 340
-  - path: src/eval/examples/
-    md5: 70fdd803300cbdc6dd76dec2148e0e0c.dir
-    size: 425678
-    nfiles: 10

dvc.yaml

@@ -1,38 +0,0 @@
-stages:
-  process_data:
-    cmd: python3 src/code/make_dataset.py src/data/raw/nyu_depth_v2_labeled.mat src/data/raw/splits.mat
-      src/data/processed
-    deps:
-    - src/code/make_dataset.py
-    - src/data/raw/nyu_depth_v2_labeled.mat
-    - src/data/raw/splits.mat
-    outs:
-    - src/data/processed/
-  train:
-    cmd: python3 src/code/training.py src/data/processed/train
-    deps:
-    - src/code/custom_data_loading.py
-    - src/code/params.yml
-    - src/code/training.py
-    - src/data/processed/train
-    outs:
-    - src/models/
-    - logs/train_params.yml:
-        cache: false
-    metrics:
-    - logs/train_metrics.csv:
-        cache: false
-  eval:
-    cmd: python3 src/code/eval.py src/data/processed/test
-    deps:
-    - src/code/params.yml
-    - src/code/custom_data_loading.py
-    - src/code/eval_metric_calculation.py
-    - src/code/eval.py
-    - src/models/model.pth
-    - src/data/processed/test
-    outs:
-    - src/eval/examples/
-    metrics:
-    - logs/test_metrics.csv:
-        cache: false

flagged/log.csv

@@ -1,4 +0,0 @@
-name,Output,timestamp
-Google,Hello Google!,2022-02-09 18:25:43.892129
-Google,Hello Google!,2022-02-09 18:25:45.867548
-Google,Hello Google!,2022-02-09 18:25:52.447643

heroku/Aptfile

@@ -1 +0,0 @@
-https://github.com/iterative/dvc/releases/download/2.9.3/dvc_2.9.3_amd64.deb

heroku/DVC-heroku-deployment.md

@@ -1,21 +0,0 @@
-We need to give Heroku the ability to pull in data from DVC upon app start up. We will install a [buildpack](https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt) that allows the installation of apt-files and then define the Aptfile that contains a path to DVC. I.e., in the CLI run:
-
-```
-heroku buildpacks:add --index 1 heroku-community/apt
-```
-
-Then in your root project folder create a file called `Aptfile` that specifies the release of DVC you want installed, https://github.com/iterative/dvc/releases/download/2.8.3/dvc_2.8.3_amd64.deb
-
-Add the following code block to your **streamlit_app.py**:
-
-```python
-import os
-
-if "DYNO" in os.environ and os.path.isdir(".dvc"):
-    os.system("dvc config core.no_scm true")
-    if os.system(f"dvc pull {model} {image}") != 0:
-        exit("dvc pull failed")
-    os.system("rm -r .dvc .apt/usr/lib/dvc")
-```
-
-Reference: [Heroku ML](https://github.com/GuilhermeBrejeiro/deploy_ML_model_Heroku_FastAPI)

heroku/Dockerfile

@@ -1,19 +0,0 @@
-FROM python:3.9.10
-
-EXPOSE 7860
-
-WORKDIR /app
-
-COPY requirements_heroku.txt requirements_heroku.txt
-
-RUN pip install -r requirements_heroku.txt
-
-COPY /app /app  
-COPY /examples /examples
-
-COPY setup.sh setup.sh
-
-#ENV GRADIO_SERVER_NAME=0.0.0.0 
-#ENV GRADIO_SERVER_PORT="$PORT"
-
-CMD ["python", "/app/app_savta.py"]

heroku/Procfile

@@ -1 +0,0 @@
-web: source setup.sh && python app/app_savta.py

heroku/heroku.yml

@@ -1,5 +0,0 @@
-build:
-  docker:
-    web: Dockerfile
-run:
-  web: python /app/app_savta.py

heroku/requirements_heroku.txt

@@ -1,15 +0,0 @@
--f https://download.pytorch.org/whl/cpu/torch_stable.html
-dvc
-fastai
-torch==1.10.2+cpu
-h5py
-opencv-python-headless
-tqdm
-numpy
-scikit-learn
-dagshub
-tables
-fastapi
-gradio
-uvicorn
-jinja2

heroku/setup.sh

@@ -1,2 +0,0 @@
-export GRADIO_SERVER_NAME=0.0.0.0 
-export GRADIO_SERVER_PORT="$PORT"

logs/test_metrics.csv

@@ -1,10 +0,0 @@
-Name,Value,Timestamp,Step
-"a1",0.6736493,1615292013951,1
-"a2",0.9112526,1615292013951,1
-"a3",0.975165,1615292013951,1
-"abs_rel",0.21377242,1615292013951,1
-"sq_rel",0.18530104,1615292013951,1
-"rmse",0.6518149,1615292013951,1
-"rmse_log",0.24310246,1615292013951,1
-"log10",0.08433308,1615292013951,1
-"silog",20.038778,1615292013951,1

logs/train_params.yml

@@ -1,25 +0,0 @@
-DAGsHubLogger: true
-Learner: <fastai.learner.Learner object at 0x7fcf56717fd0>
-ParamScheduler: true
-ProgressCallback: true
-Recorder: {add_time: true, train_metrics: false, valid_metrics: true}
-TrainEvalCallback: true
-batch per epoch: 159
-batch size: 4
-dataset.tfms: '[Pipeline: PILBase.create, Pipeline: get_y_fn -> PILBase.create]'
-device: cuda
-dls.after_batch: "Pipeline: IntToFloatTensor -- {'div': 255.0, 'div_mask': 1} -> Normalize\
-  \ -- {'mean': tensor([[[[0.4850]],\n\n         [[0.4560]],\n\n         [[0.4060]]]],\
-  \ device='cuda:0'), 'std': tensor([[[[0.2290]],\n\n         [[0.2240]],\n\n    \
-  \     [[0.2250]]]], device='cuda:0'), 'axes': (0, 2, 3)}"
-dls.after_item: 'Pipeline: ToTensor'
-dls.before_batch: 'Pipeline: '
-frozen: false
-frozen idx: 0
-input 1 dim 1: 4
-input 1 dim 2: 3
-input 1 dim 3: 480
-input 1 dim 4: 640
-model parameters: 41221268
-n_inp: 1
-success: true

models/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bc9733d71d12a9cd8996f72959b280691fddd8b503be6eef60b56f022492196
+size 494927324

requirements.txt

@@ -1,7 +1,3 @@
-# -------- data & version locks -----------
-# Drop heavy DVC stack, switch to DagsHub CLI for downloads
-dagshub==0.3.34            # CLI + streaming support
-pathspec==0.9.0
 
 # -------- deep-learning stack ------------
 fastai==2.7.14             # works with torch < 2.3

run_dev_env.sh

@@ -1,7 +0,0 @@
-docker run -d \
-    -p 8080:8080 \
-    --name "dags-ml-workspace" -v "/${PWD}:/workspace" \
-    --env AUTHENTICATE_VIA_JUPYTER="dagshub_savta" \
-    --shm-size 2G \
-    --restart always \
-    dagshub/ml-workspace-minimal:latest

src/.gitignore

@@ -1,2 +0,0 @@
-/models
-/eval

src/code/custom_data_loading.py

@@ -1,51 +0,0 @@
-import yaml
-from fastai.vision.all import \
-    DataLoaders, \
-    delegates, \
-    DataBlock, \
-    ImageBlock, \
-    PILImage, \
-    PILImageBW, \
-    RandomSplitter, \
-    Path, \
-    get_files
-
-
-class ImageImageDataLoaders(DataLoaders):
-    """Basic wrapper around several `DataLoader`s with factory methods for Image to Image problems"""
-    @classmethod
-    @delegates(DataLoaders.from_dblock)
-    def from_label_func(cls, path, filenames, label_func, valid_pct=0.2, seed=None, item_transforms=None,
-                        batch_transforms=None, **kwargs):
-        """Create from list of `fnames` in `path`s with `label_func`."""
-        datablock = DataBlock(blocks=(ImageBlock(cls=PILImage), ImageBlock(cls=PILImageBW)),
-                              get_y=label_func,
-                              splitter=RandomSplitter(valid_pct, seed=seed),
-                              item_tfms=item_transforms,
-                              batch_tfms=batch_transforms)
-        res = cls.from_dblock(datablock, filenames, path=path, **kwargs)
-        return res
-
-
-def get_y_fn(x):
-    y = str(x.absolute()).replace('.jpg', '_depth.png')
-    y = Path(y)
-
-    return y
-
-
-def create_data(data_path):
-    with open(r"./src/code/params.yml") as f:
-        params = yaml.safe_load(f)
-
-    filenames = get_files(data_path, extensions='.jpg')
-    if len(filenames) == 0:
-        raise ValueError("Could not find any files in the given path")
-    dataset = ImageImageDataLoaders.from_label_func(data_path,
-                                                    seed=int(params['seed']),
-                                                    bs=int(params['batch_size']),
-                                                    num_workers=int(params['num_workers']),
-                                                    filenames=filenames,
-                                                    label_func=get_y_fn)
-
-    return dataset

src/code/eval.py

@@ -1,56 +0,0 @@
-import sys
-import yaml
-import torch
-from torchvision import transforms
-from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat, get_files, L, PILImageBW
-from custom_data_loading import create_data
-from eval_metric_calculation import compute_eval_metrics
-from dagshub import dagshub_logger
-from tqdm import tqdm
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("usage: %s <test_data_path>" % sys.argv[0], file=sys.stderr)
-        sys.exit(0)
-
-    with open(r"./src/code/params.yml") as f:
-        params = yaml.safe_load(f)
-
-    data_path = Path(sys.argv[1])
-    data = create_data(data_path)
-
-    arch = {'resnet34': resnet34}
-    loss = {'MSELossFlat': MSELossFlat()}
-
-    learner = unet_learner(data,
-                           arch.get(params['architecture']),
-                           n_out=int(params['num_outs']),
-                           loss_func=loss.get(params['loss_func']),
-                           path='src/',
-                           model_dir='models')
-    learner = learner.load('model')
-
-    filenames = get_files(Path(data_path), extensions='.jpg')
-    test_files = L([Path(i) for i in filenames])
-
-    for i, sample in tqdm(enumerate(test_files.items),
-                          desc="Predicting on test images",
-                          total=len(test_files.items)):
-        pred = learner.predict(sample)[0]
-        pred = PILImageBW.create(pred).convert('L')
-        pred.save("src/eval/" + str(sample.stem) + "_pred.png")
-        if i < 10:
-            pred.save("src/eval/examples/" + str(sample.stem) + "_pred.png")
-
-    print("Calculating metrics...")
-    metrics = compute_eval_metrics(test_files)
-
-    with dagshub_logger(
-            metrics_path="logs/test_metrics.csv",
-            should_log_hparams=False
-    ) as logger:
-        # Metric logging
-        logger.log_metrics(metrics)
-
-    print("Evaluation Done!")

src/code/eval_metric_calculation.py

@@ -1,79 +0,0 @@
-import numpy as np
-from PIL import Image
-from tqdm import tqdm
-
-def compute_errors(target, prediction):
-    thresh = np.maximum((target / prediction), (prediction / target))
-    a1 = (thresh < 1.25).mean()
-    a2 = (thresh < 1.25 ** 2).mean()
-    a3 = (thresh < 1.25 ** 3).mean()
-
-    abs_rel = np.mean(np.abs(target - prediction) / target)
-    sq_rel = np.mean(((target - prediction) ** 2) / target)
-
-    rmse = (target - prediction) ** 2
-    rmse = np.sqrt(rmse.mean())
-
-    rmse_log = (np.log(target) - np.log(prediction)) ** 2
-    rmse_log = np.sqrt(rmse_log.mean())
-
-    err = np.log(prediction) - np.log(target)
-    silog = np.sqrt(np.mean(err ** 2) - np.mean(err) ** 2) * 100
-
-    log_10 = (np.abs(np.log10(target) - np.log10(prediction))).mean()
-
-    return a1, a2, a3, abs_rel, sq_rel, rmse, rmse_log, silog, log_10
-
-
-def compute_eval_metrics(test_files):
-    min_depth_eval = 1e-3
-    max_depth_eval = 10
-
-    num_samples = len(test_files)
-
-    a1 = np.zeros(num_samples, np.float32)
-    a2 = np.zeros(num_samples, np.float32)
-    a3 = np.zeros(num_samples, np.float32)
-    abs_rel = np.zeros(num_samples, np.float32)
-    sq_rel = np.zeros(num_samples, np.float32)
-    rmse = np.zeros(num_samples, np.float32)
-    rmse_log = np.zeros(num_samples, np.float32)
-    silog = np.zeros(num_samples, np.float32)
-    log10 = np.zeros(num_samples, np.float32)
-
-    for i in tqdm(range(num_samples), desc="Calculating metrics for test data", total=num_samples):
-        sample_path = test_files[i]
-        target_path = str(sample_path.parent/(sample_path.stem + "_depth.png"))
-        pred_path = "src/eval/" + str(sample_path.stem) + "_pred.png"
-
-        target_image = Image.open(target_path)
-        pred_image = Image.open(pred_path)
-
-        target = np.asarray(target_image)
-        pred = np.asarray(pred_image)
-
-        target = target / 25.0
-        pred = pred / 25.0
-
-        pred[pred < min_depth_eval] = min_depth_eval
-        pred[pred > max_depth_eval] = max_depth_eval
-        pred[np.isinf(pred)] = max_depth_eval
-
-        target[np.isinf(target)] = 0
-        target[np.isnan(target)] = 0
-
-        valid_mask = np.logical_and(target > min_depth_eval, target < max_depth_eval)
-
-        a1[i], a2[i], a3[i], abs_rel[i], sq_rel[i], rmse[i], rmse_log[i], silog[i], log10[i] = \
-            compute_errors(target[valid_mask], pred[valid_mask])
-
-    print("{:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}".format(
-        'd1', 'd2', 'd3', 'AbsRel', 'SqRel', 'RMSE', 'RMSElog', 'SILog', 'log10'))
-    print("{:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}".format(
-        a1.mean(), a2.mean(), a3.mean(),
-        abs_rel.mean(), sq_rel.mean(), rmse.mean(), rmse_log.mean(), silog.mean(), log10.mean()))
-
-    return dict(a1=a1.mean(), a2=a2.mean(), a3=a3.mean(),
-                abs_rel=abs_rel.mean(), sq_rel=sq_rel.mean(),
-                rmse=rmse.mean(), rmse_log=rmse_log.mean(),
-                log10=log10.mean(), silog=silog.mean())

src/code/make_dataset.py

@@ -1,121 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#######################################################################################
-# The MIT License
-
-# Copyright (c) 2014       Hannes Schulz, University of Bonn  <schulz@ais.uni-bonn.de>
-# Copyright (c) 2013       Benedikt Waldvogel, University of Bonn <mail@bwaldvogel.de>
-# Copyright (c) 2008-2009  Sebastian Nowozin                       <nowozin@gmail.com>
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-# 
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-# 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#######################################################################################
-#
-# See https://github.com/deeplearningais/curfil/wiki/Training-and-Prediction-with-the-NYU-Depth-v2-Dataset
-
-
-"""Helper script to convert the NYU Depth v2 dataset Matlab file into a set of PNG and JPEG images.
-Receives 3 Files from argparse:
-<h5_file> - Contains the original images, depths maps, and scene types
-<train_test_split> - contains two numpy arrays with the index of the
-                    images based on the split to train and test sets.
-<out_folder> - Name of the folder to save the original and depth images.
-
-Every image in the DB will have it's twine B&W image that indicates the depth
-in the image. the images will be read, converted by the convert_image function
-and finally saved to path based on train test split and Scene types.
-"""
-
-from __future__ import print_function
-
-import h5py
-import numpy as np
-import os
-import scipy.io
-import sys
-import cv2
-from tqdm import tqdm
-
-
-def convert_image(index, depth_map, img, output_folder):
-    """Processes data images and depth maps
-    :param index: int, image index
-    :param depth_map: numpy array, image depth - 2D array.
-    :param img: numpy array, the original RGB image - 3D array.
-    :param output_folder: path to save the image in.
-
-    Receives an image with it's relevant depth map.
-    Normalizes the depth map, and adds a 7 px boundary to the original image.
-    Saves both image and depth map to the appropriate processed data folder.
-    """
-
-    # Normalize the depth image
-    # normalized_depth = cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX)
-    img_depth = depth_map * 25.0
-    cv2.imwrite("%s/%05d_depth.png" % (output_folder, index), img_depth)
-
-    # Adding black frame to original image
-    img = img[:, :, ::-1]  # Flipping the image from RGB to BGR for opencv
-    image_black_boundary = np.zeros(img.shape, dtype=np.uint8)
-    image_black_boundary[7:image_black_boundary.shape[0] - 6, 7:image_black_boundary.shape[1] - 6, :] = \
-        img[7:img.shape[0] - 6, 7:img.shape[1] - 6, :]
-    cv2.imwrite("%s/%05d.jpg" % (output_folder, index), image_black_boundary)
-
-
-if __name__ == "__main__":
-
-    # Check if got all needed input for argparse
-    if len(sys.argv) != 4:
-        print("usage: %s <h5_file> <train_test_split> <out_folder>" % sys.argv[0], file=sys.stderr)
-        sys.exit(0)
-
-    # load arguments to variables
-    h5_file = h5py.File(sys.argv[1], "r")
-    train_test = scipy.io.loadmat(sys.argv[2])  # h5py is not able to open that file. but scipy is
-    out_folder = sys.argv[3]
-
-    # Extract images *indexes* for train and test data sets
-    test_images = set([int(x) for x in train_test["testNdxs"]])
-    train_images = set([int(x) for x in train_test["trainNdxs"]])
-    print("%d training images" % len(train_images))
-    print("%d test images" % len(test_images))
-
-    # Grayscale
-    depth = h5_file['depths']
-    print("Reading", sys.argv[1])
-    images = h5_file['images']  # (num_channels, height, width)
-
-    # Extract all sceneTypes per image - "office", "classroom", etc.
-    scenes = [u''.join(chr(c[0]) for c in h5_file[obj_ref]) for obj_ref in h5_file['sceneTypes'][0]]
-
-    for i, image in tqdm(enumerate(images), desc="Processing images", total=len(images)):
-        idx = int(i) + 1
-        if idx in train_images:
-            train_test = "train"
-        else:
-            assert idx in test_images, "index %d neither found in training set nor in test set" % idx
-            train_test = "test"
-
-        # Create path to save image in
-        folder = "%s/%s/%s" % (out_folder, train_test, scenes[i])
-        if not os.path.exists(folder):
-            os.makedirs(folder)
-
-        convert_image(i, depth[i, :, :].T, image.T, folder)
-
-    print("Finished")

src/code/params.yml

@@ -1,13 +0,0 @@
-seed: 42
-data: nyu_depth_v2
-batch_size: 4
-num_workers: 0
-weight_decay: 1e-2
-learning_rate: 1e-3
-epochs: 50
-num_outs: 3
-source_dir: src
-model_dir: models
-architecture: resnet34
-loss_func: MSELossFlat
-train_metric: rmse

src/code/training.py

@@ -1,44 +0,0 @@
-"""Trains or fine-tunes a model for the task of monocular depth estimation
-Receives 1 arguments from argparse:
-  <data_path> - Path to the dataset which is split into 2 folders - train and test.
-"""
-import sys
-import yaml
-from fastai.vision.all import unet_learner, Path, resnet34, rmse, MSELossFlat
-from custom_data_loading import create_data
-from dagshub.fastai import DAGsHubLogger
-
-
-if __name__ == "__main__":
-    # Check if got all needed input for argparse
-    if len(sys.argv) != 2:
-        print("usage: %s <data_path>" % sys.argv[0], file=sys.stderr)
-        sys.exit(0)
-
-    with open(r"./src/code/params.yml") as f:
-        params = yaml.safe_load(f)
-
-    data = create_data(Path(sys.argv[1]))
-
-    metrics = {'rmse': rmse}
-    arch = {'resnet34': resnet34}
-    loss = {'MSELossFlat': MSELossFlat()}
-
-    learner = unet_learner(data,
-                           arch.get(params['architecture']),
-                           metrics=metrics.get(params['train_metric']),
-                           wd=float(params['weight_decay']),
-                           n_out=int(params['num_outs']),
-                           loss_func=loss.get(params['loss_func']),
-                           path=params['source_dir'],
-                           model_dir=params['model_dir'],
-                           cbs=DAGsHubLogger(
-                               metrics_path="logs/train_metrics.csv",
-                               hparams_path="logs/train_params.yml"))
-
-    print("Training model...")
-    learner.fine_tune(epochs=int(params['epochs']),
-                      base_lr=float(params['learning_rate']))
-    print("Saving model...")
-    learner.save('model')
-    print("Done!")

src/data/.gitignore

@@ -1 +0,0 @@
-/processed

src/data/raw/.gitignore

@@ -1,2 +0,0 @@
-/nyu_depth_v2_labeled.mat
-/splits.mat

src/data/raw/nyu_depth_v2_labeled.mat.dvc

@@ -1,9 +0,0 @@
-md5: d27a0ba6c898f981797a3388c26c2d0f
-frozen: true
-deps:
-- etag: '"b125b2b1-5aa5b95864fc7"'
-  path: http://horatio.cs.nyu.edu/mit/silberman/nyu_depth_v2/nyu_depth_v2_labeled.mat
-outs:
-- md5: 520609c519fba3ba5ac58c8fefcc3530
-  path: nyu_depth_v2_labeled.mat
-  size: 2972037809

src/data/raw/splits.mat.dvc

@@ -1,9 +0,0 @@
-md5: 26011289311c18b92781de66654223a4
-frozen: true
-deps:
-- etag: '"a42-4cb6a5fad2fc0"'
-  path: http://horatio.cs.nyu.edu/mit/silberman/indoor_seg_sup/splits.mat
-outs:
-- md5: 08e3c3aea27130ac7c01ffd739a4535f
-  path: splits.mat
-  size: 2626