add define

.gitattributes

@@ -32,3 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+**/*.gif filter=lfs diff=lfs merge=lfs -text
+**/*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+*.pyc
+.idea
+*ZZZ*
+sandbox
+__pycache__
+wandb
+temp
+code

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "externals/camviz"]
+	path = externals/camviz
+	url = [email protected]:TRI-ML/camviz.git

LICENSE.md

@@ -0,0 +1,409 @@
+Attribution-NonCommercial 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and
+conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  j. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  k. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  l. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+
+

Makefile

@@ -0,0 +1,56 @@
+PROJECT ?= vidar
+WORKSPACE ?= /workspace/$(PROJECT)
+DOCKER_IMAGE ?= ${PROJECT}:latest
+
+SHMSIZE ?= 444G
+WANDB_MODE ?= run
+DOCKER_OPTS := \
+			--name ${PROJECT} \
+			--rm -it \
+			--shm-size=${SHMSIZE} \
+			-e AWS_DEFAULT_REGION \
+			-e AWS_ACCESS_KEY_ID \
+			-e AWS_SECRET_ACCESS_KEY \
+			-e WANDB_API_KEY \
+			-e WANDB_ENTITY \
+			-e WANDB_MODE \
+			-e HOST_HOSTNAME= \
+			-e OMP_NUM_THREADS=1 -e KMP_AFFINITY="granularity=fine,compact,1,0" \
+			-e OMPI_ALLOW_RUN_AS_ROOT=1 \
+			-e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \
+			-e NCCL_DEBUG=VERSION \
+            -e DISPLAY=${DISPLAY} \
+            -e XAUTHORITY \
+            -e NVIDIA_DRIVER_CAPABILITIES=all \
+			-v ~/.aws:/root/.aws \
+			-v /root/.ssh:/root/.ssh \
+			-v ~/.cache:/root/.cache \
+			-v /dev/null:/dev/raw1394 \
+			-v /mnt/fsx/tmp:/tmp \
+			-v /tmp/.X11-unix/X0:/tmp/.X11-unix/X0 \
+			-v /var/run/docker.sock:/var/run/docker.sock \
+			-v /home/jiadingfang/datasets:/data \
+			-v ${PWD}:${WORKSPACE} \
+			-w ${WORKSPACE} \
+			--privileged \
+			--ipc=host \
+			--network=host
+
+NGPUS=$(shell nvidia-smi -L | wc -l)
+
+all: clean
+
+clean:
+	find . -name "*.pyc" | xargs rm -f && \
+	find . -name "__pycache__" | xargs rm -rf
+
+docker-build:
+	docker build \
+		-f docker/Dockerfile \
+		-t ${DOCKER_IMAGE} .
+
+docker-interactive: docker-build
+	nvidia-docker run ${DOCKER_OPTS} ${DOCKER_IMAGE} bash
+
+docker-run: docker-build
+	nvidia-docker run ${DOCKER_OPTS} ${DOCKER_IMAGE} bash -c "${COMMAND}"

configs/overfit/ddad_tiny.yaml

@@ -0,0 +1,31 @@
+wrapper:
+    recipe: wrapper|default
+    max_epochs: 1
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/pose_net|default
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|ddad_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+    pose:
+        recipe: optimizers|adam_20_05
+datasets:
+    train:
+        recipe: datasets/ddad_tiny|train_selfsup_front
+    validation:
+        recipe: datasets/ddad_tiny|validation_front
+

configs/overfit/generic.yaml

@@ -0,0 +1,27 @@
+wrapper:
+    recipe: wrapper|default
+    max_epochs: 1
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/pose_net|default
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+    pose:
+        recipe: optimizers|adam_20_05
+datasets:
+    train:
+        recipe: datasets/generic|default_train
+    validation:
+        recipe: datasets/generic|default_validation

configs/overfit/kitti_tiny.yaml

@@ -0,0 +1,31 @@
+wrapper:
+    recipe: wrapper|default
+    max_epochs: 1
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/pose_net|default
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+    pose:
+        recipe: optimizers|adam_20_05
+datasets:
+    train:
+        recipe: datasets/kitti_tiny|train_selfsup_mr
+    validation:
+        recipe: datasets/kitti_tiny|validation_mr
+

configs/papers/define/scannet_temporal_test.yaml

@@ -0,0 +1,79 @@
+wrapper:
+    seed: 42
+    min_epochs: 0
+    max_epochs: 0
+    find_unused_parameters: True
+    flip_lr_prob: 0.5
+    validate_first: True
+    validate_flipped: False
+    sync_batch_norm: True
+evaluation:
+    rgb:
+        only_first: False
+    depth:
+        crop: ''
+        min_depth: 0.2
+        max_depth: 10.0
+        scale_output: resize
+        median_scaling: True
+        post_process: False
+arch:
+    model:
+        checkpoint: /data/vidar/models/scannet_full.ckpt
+        file: perceiver/DefineGenericModel
+        use_pose_noise: []
+        use_virtual_cameras: []
+        virtual_cameras_eval: False
+        use_virtual_rgb: False
+        augment_canonical: False
+        encode_train: all
+        encode_eval: all
+        decode_train: all
+        decode_eval: all
+        decode_encodes: False
+        task_weights: [1.0,1.0]
+        scale_loss: True
+        sample_decoded_queries: 0.0
+        network:
+            tasks: [depth]
+            depth_range: [0.1,10.]
+            image_shape: [128,192]
+            num_bands_orig: 20
+            num_bands_dirs: 10
+            max_resolution_orig: 60
+            max_resolution_dirs: 60
+            to_world: True
+            d_latents: 512
+            num_latents: 1024
+            hidden_dropout_prob: 0.25
+            num_cross_attention_heads: 1
+            num_self_attends_per_block: 8
+            num_self_attention_heads: 8
+            decoder_num_heads: 1
+            rgb_feat_dim: 960
+            rgb_feat_type: resnet_all
+            downsample_encoder: 4
+            downsample_decoder: 4
+            upsample_convex: 'convex'
+            encoder_with_rgb: True
+            decoder_with_rgb: False
+            output_mode: inv_depth
+            sample_encoding_rays: 0
+            with_monodepth: False
+datasets:
+    validation:
+        name: [ScanNetTemporal]
+        path: [/data/vidar/scannet_processed]
+        split: [test]
+        context: [-2,2]
+        stride: [10]
+        labels: [depth,pose]
+        cameras: [[0]]
+        dataloader:
+            batch_size: 1
+            pin_memory: True
+            num_workers: 4
+        augmentation:
+            resize: [128,192]
+            resize_supervision: True
+            preserve_depth: True

configs/papers/define/scannet_temporal_test_context_1.yaml

@@ -0,0 +1,79 @@
+wrapper:
+    seed: 42
+    min_epochs: 0
+    max_epochs: 0
+    find_unused_parameters: True
+    flip_lr_prob: 0.5
+    validate_first: True
+    validate_flipped: False
+    sync_batch_norm: True
+evaluation:
+    rgb:
+        only_first: False
+    depth:
+        crop: ''
+        min_depth: 0.2
+        max_depth: 10.0
+        scale_output: resize
+        median_scaling: True
+        post_process: False
+arch:
+    model:
+        checkpoint: /data/vidar/models/scannet_full.ckpt
+        file: perceiver/DefineGenericModel
+        use_pose_noise: []
+        use_virtual_cameras: []
+        virtual_cameras_eval: False
+        use_virtual_rgb: False
+        augment_canonical: False
+        encode_train: all
+        encode_eval: all
+        decode_train: all
+        decode_eval: all
+        decode_encodes: False
+        task_weights: [1.0,1.0]
+        scale_loss: True
+        sample_decoded_queries: 0.0
+        network:
+            tasks: [depth]
+            depth_range: [0.1,10.]
+            image_shape: [128,192]
+            num_bands_orig: 20
+            num_bands_dirs: 10
+            max_resolution_orig: 60
+            max_resolution_dirs: 60
+            to_world: True
+            d_latents: 512
+            num_latents: 1024
+            hidden_dropout_prob: 0.25
+            num_cross_attention_heads: 1
+            num_self_attends_per_block: 8
+            num_self_attention_heads: 8
+            decoder_num_heads: 1
+            rgb_feat_dim: 960
+            rgb_feat_type: resnet_all
+            downsample_encoder: 4
+            downsample_decoder: 4
+            upsample_convex: 'convex'
+            encoder_with_rgb: True
+            decoder_with_rgb: False
+            output_mode: inv_depth
+            sample_encoding_rays: 0
+            with_monodepth: False
+datasets:
+    validation:
+        name: [ScanNetTemporal]
+        path: [/data/vidar/scannet_processed]
+        split: [test]
+        context: [1]
+        stride: [10]
+        labels: [depth,pose]
+        cameras: [[0]]
+        dataloader:
+            batch_size: 1
+            pin_memory: True
+            num_workers: 4
+        augmentation:
+            resize: [128,192]
+            resize_supervision: True
+            preserve_depth: True

configs/papers/depthformer/inference_kitti.yaml

@@ -0,0 +1,31 @@
+wrapper:
+    recipe: wrapper|default
+arch:
+    model:
+        file: depth/DepthFormerModel
+        checkpoint: /data/vidar/models/papers/final/DepthFormer_MR_selfsup_KITTI.ckpt
+        warp_context: [-1,1]
+        match_context: [-1]
+        motion_masking: True
+        matching_augmentation: False
+        freeze_teacher_and_pose: 40
+    networks:
+        transformer:
+            recipe: networks/transformer|depthformer
+        mono_depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        multi_depth:
+            recipe: networks/multi_depth_res_net|depthformer
+        pose:
+            recipe: networks/pose_net|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+datasets:
+    validation:
+        recipe: datasets/kitti|validation_mr
+        labels: [depth,pose]
+        context: [-1,1]
+save:
+    recipe: save|depth_splitname

configs/papers/fsm/inference_ddad.yaml

@@ -0,0 +1,19 @@
+wrapper:
+    recipe: wrapper|default
+arch:
+    model:
+        file: depth/FSMModel
+        checkpoint: /data/vidar/models/papers/final/FSM_MR_6cams_DDAD.ckpt
+    networks:
+        depth:
+            recipe: networks/focal_depth_res_net|fsm_ddad
+        pose:
+            recipe: networks/conv_pose_net|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|ddad_resize
+datasets:
+    validation:
+        recipe: datasets/ddad|validation_6cams
+save:
+    recipe: save|depth_splitname

configs/papers/packnet/inference_packnet.yaml

@@ -0,0 +1,20 @@
+wrapper:
+    recipe: wrapper|default
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        checkpoint: /data/vidar/models/papers/final/PackNet_MR_selfsup_KITTI.ckpt
+    networks:
+        depth:
+            recipe: networks/packnet|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/conv_pose_net|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+datasets:
+    validation:
+        recipe: datasets/kitti|validation_mr
+save:
+    recipe: save|depth_splitname

configs/papers/packnet/inference_resnet.yaml

@@ -0,0 +1,20 @@
+wrapper:
+    recipe: wrapper|default
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        checkpoint: /data/vidar/models/papers/final/ResNet18_MR_selfsup_KITTI.ckpt
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/pose_net|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+datasets:
+    validation:
+        recipe: datasets/kitti|validation_mr
+save:
+    recipe: save|depth_splitname

configs/papers/selfcalib/ds_euroc.yaml

@@ -0,0 +1,40 @@
+wrapper:
+    recipe: wrapper|default
+    validate_first: False
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        use_gt_intrinsics: False
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,80.0]
+        pose:
+            recipe: networks/pose_net|default
+        intrinsics:
+            file: intrinsics/IntrinsicsNet
+            camera_model: 'DS'
+            shape: [256, 384]
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    pose:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    intrinsics:
+        recipe: optimizers|adam_20_05
+        lr: 0.01
+datasets:
+    train:
+        recipe: datasets/euroc|train_selfsup
+    validation:
+        recipe: datasets/euroc|validation

configs/papers/selfcalib/eucm_euroc.yaml

@@ -0,0 +1,40 @@
+wrapper:
+    recipe: wrapper|default
+    validate_first: False
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        use_gt_intrinsics: False
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,80.0]
+        pose:
+            recipe: networks/pose_net|default
+        intrinsics:
+            file: intrinsics/IntrinsicsNet
+            camera_model: 'EUCM'
+            shape: [256, 384]
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    pose:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    intrinsics:
+        recipe: optimizers|adam_20_05
+        lr: 0.01
+datasets:
+    train:
+        recipe: datasets/euroc|train_selfsup
+    validation:
+        recipe: datasets/euroc|validation

configs/papers/selfcalib/ucm_euroc.yaml

@@ -0,0 +1,40 @@
+wrapper:
+    recipe: wrapper|default
+    validate_first: False
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        use_gt_intrinsics: False
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,80.0]
+        pose:
+            recipe: networks/pose_net|default
+        intrinsics:
+            file: intrinsics/IntrinsicsNet
+            camera_model: 'UCM'
+            shape: [256, 384]
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    pose:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    intrinsics:
+        recipe: optimizers|adam_20_05
+        lr: 0.01
+datasets:
+    train:
+        recipe: datasets/euroc|train_selfsup
+    validation:
+        recipe: datasets/euroc|validation

configs/papers/selfcalib/ucm_kitti.yaml

@@ -0,0 +1,44 @@
+wrapper:
+    recipe: wrapper|default
+    validate_first: False
+arch:
+    model:
+        file: depth/SelfSupervisedModel
+        use_gt_intrinsics: False
+    networks:
+        depth:
+            recipe: networks/mono_depth_res_net|default
+            depth_range: [0.1,100.0]
+        pose:
+            recipe: networks/pose_net|default
+        intrinsics:
+            file: intrinsics/IntrinsicsNet
+            camera_model: 'UCM'
+            shape: [192, 640]
+    losses:
+        reprojection:
+            recipe: losses/reprojection|default
+        smoothness:
+            recipe: losses/smoothness|default
+evaluation:
+    depth:
+        recipe: evaluation/depth|kitti_resize
+optimizers:
+    depth:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    pose:
+        recipe: optimizers|adam_20_05
+        lr: 0.0002
+    intrinsics:
+        recipe: optimizers|adam_20_05
+        lr: 0.01
+datasets:
+    train:
+        recipe: datasets/kitti|train_selfsup_mr
+        dataloader:
+            batch_size: 8
+    validation:
+        recipe: datasets/kitti|validation_mr
+        dataloader:
+            batch_size: 1

configs/recipes/checkpoint.yaml

@@ -0,0 +1,8 @@
+default:
+    folder: /data/vidar/checkpoints
+    s3_bucket: miru-us-east-1/vidar/checkpoints
+    save_code: True
+    keep_top: 5
+default_local:
+    folder: /data/vidar/checkpoints
+    keep_top: 5

configs/recipes/datasets/ddad.yaml

@@ -0,0 +1,32 @@
+train_selfsup_6cams:
+    name: [Ouroboros]
+    path: [/data/vidar/DDAD/ddad.json]
+    split: [train]
+    augmentation:
+        jittering: [0.2, 0.2, 0.2, 0.05]
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: [-1,1]
+    labels: [depth]
+    cameras: [[1],[5],[6],[7],[8],[9]]
+    depth_type: [lidar]
+    repeat: [100]
+validation_6cams:
+    name: [Ouroboros]
+    path: [/data/vidar/DDAD/ddad.json]
+    split: [val]
+    augmentation:
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: []
+    labels: [depth]
+    cameras: [[1],[5],[6],[7],[8],[9]]
+    depth_type: [lidar]
+
+

configs/recipes/datasets/ddad_tiny.yaml

@@ -0,0 +1,32 @@
+train_selfsup_front:
+    name: [Ouroboros]
+    path: [/data/vidar/DDAD_tiny/ddad_tiny.json]
+    split: [train]
+    augmentation:
+        jittering: [0.2, 0.2, 0.2, 0.05]
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: [-1,1]
+    labels: [depth]
+    cameras: [[1]]
+    depth_type: [lidar]
+    repeat: [100]
+validation_front:
+    name: [Ouroboros]
+    path: [/data/vidar/DDAD_tiny/ddad_tiny.json]
+    split: [train]
+    augmentation:
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: []
+    labels: [depth]
+    cameras: [[1]]
+    depth_type: [lidar]
+
+

configs/recipes/datasets/euroc.yaml

@@ -0,0 +1,30 @@
+train_selfsup:
+    name: [EUROC]
+    path: [/data/vidar/euroc/euroc_cam/cam0]
+    augmentation:
+        jittering: [0.2, 0.2, 0.2, 0.05]
+        resize: [256, 384]
+    dataloader:
+        batch_size: 16
+        pin_memory: True
+        num_workers: 16
+    context: [-1, 1]
+    strides: [[49999872, 50000128]]
+    cameras: [[0]]
+    split: [euroc-train]
+    labels: []
+    repeat: [1]
+validation:
+    name: [EUROC]
+    path: [/data/vidar/euroc/euroc_has_depth/V2_01_easy_has_depth/mav0/]
+    augmentation:
+        resize: [256, 384]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    split: [euroc-val]
+    labels: [depth]
+    context: []
+    strides: [[0, 0]]
+    cameras: [[0]]

configs/recipes/datasets/generic.yaml

@@ -0,0 +1,30 @@
+default_train:
+    name: [Generic]
+    path: [data/generic]
+    split: ['']
+    context: [-1,1]
+    cameras: [[0]]
+    labels: []
+    extension: [jpg]
+    repeat: [100]
+    augmentation:
+        jittering: [0.2, 0.2, 0.2, 0.05]
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+default_validation:
+    name: [Generic]
+    path: [data/generic]
+    split: ['']
+    context: []
+    cameras: [[0]]
+    labels: []
+    extension: [jpg]
+    augmentation:
+        resize: [384, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16

configs/recipes/datasets/kitti.yaml

@@ -0,0 +1,31 @@
+train_selfsup_mr:
+    name: [KITTI]
+    path: [/data/vidar/KITTI_raw]
+    split: [data_splits/eigen_zhou_files.txt]
+    augmentation:
+        jittering: [0.2, 0.2, 0.2, 0.05]
+        resize: [192, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: [-1,1]
+    labels: []
+    cameras: [[0]]
+    single_intrinsics: [True]
+    repeat: [1]
+validation_mr:
+    name: [KITTI]
+    path: [/data/vidar/KITTI_raw]
+    split: [data_splits/eigen_test_files.txt]
+    augmentation:
+        resize: [192, 640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: []
+    labels: [depth]
+    cameras: [[0]]
+    single_intrinsics: [True]
+    depth_type: [velodyne,groundtruth]

configs/recipes/datasets/kitti_tiny.yaml

@@ -0,0 +1,32 @@
+train_selfsup_mr:
+    name: [KITTI]
+    path: [/data/vidar/KITTI_tiny]
+    split: [kitti_tiny.txt]
+    augmentation:
+        jittering: [0.2,0.2,0.2,0.05]
+        resize: [192,640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: [-1,1]
+    labels: []
+    cameras: [[0]]
+    single_intrinsics: [True]
+    repeat: [100]
+    depth_type: [velodyne]
+validation_mr:
+    name: [KITTI]
+    path: [/data/vidar/KITTI_tiny]
+    split: [kitti_tiny.txt]
+    augmentation:
+        resize: [192,640]
+    dataloader:
+        batch_size: 1
+        pin_memory: True
+        num_workers: 16
+    context: []
+    labels: [depth]
+    cameras: [[0]]
+    single_intrinsics: [True]
+    depth_type: [velodyne]

configs/recipes/datasets/vkitti2.yaml

@@ -0,0 +1,16 @@
+tiny:
+    name: [VKITTI2]
+    path: [/data/vidar/VKITTI2_tiny]
+    split: ['']
+    augmentation:
+      resize: [192, 640]
+      resize_supervision: True
+    dataloader:
+      batch_size: 1
+      pin_memory: True
+      num_workers: 16
+    context: [-1,1]
+    labels: [depth,pose]
+    labels_context: [depth,pose]
+    cameras: [[0]]
+    depth_type: [zbuffer]

configs/recipes/evaluation/depth.yaml

@@ -0,0 +1,21 @@
+kitti_resize:
+    crop: garg
+    min_depth: 0.0
+    max_depth: 80.0
+    scale_output: resize
+    median_scaling: True
+    post_process: True
+kitti_crop:
+    crop: garg
+    min_depth: 0.0
+    max_depth: 80.0
+    scale_output: top-center
+    median_scaling: True
+    post_process: True
+ddad_resize:
+    crop: ''
+    min_depth: 0.0
+    max_depth: 200.0
+    scale_output: resize
+    median_scaling: True
+    post_process: True

configs/recipes/losses/reprojection.yaml

@@ -0,0 +1,9 @@
+default:
+    file: ReprojectionLoss
+    automasking: True
+    reprojection_reduce_op: min
+    jitter_identity_reprojection: 0.00001
+    photometric:
+        file: PhotometricLoss
+        weight: 1.0
+        alpha: 0.85

configs/recipes/losses/smoothness.yaml

@@ -0,0 +1,5 @@
+default:
+    file: SmoothnessLoss
+    normalize: True
+    weight: 0.0001
+    gamma: 0.5

configs/recipes/losses/supervised_depth.yaml

@@ -0,0 +1,40 @@
+huber:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: huber
+    mask_sparse: True
+abs_rel:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: abs_rel
+    mask_sparse: True
+l1log:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: l1log
+    mask_sparse: True
+l1:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: l1
+    mask_sparse: True
+mse:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: mse
+    mask_sparse: True
+rmse:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: rmse
+    mask_sparse: True
+mixture:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: mixture
+    mask_sparse: True
+cross_entropy:
+    file: SupervisedDepthLoss
+    gamma: 0.5
+    method: cross_entropy
+    mask_sparse: True

configs/recipes/networks/conv_pose_net.yaml

@@ -0,0 +1,5 @@
+default:
+    file: pose/ConvPoseNet
+    version: 18
+    pretrained: True
+    num_rgb_in: 2

configs/recipes/networks/focal_depth_res_net.yaml

@@ -0,0 +1,12 @@
+fsm_ddad:
+    file: depth/FocalDepthResNet
+    encoder:
+        version: 18
+        pretrained: True
+        num_rgb_in: 1
+    decoder:
+        use_skips: True
+        activation: sigmoid
+        num_ch_out: 1
+    min_depth: 0.005
+    max_depth: 0.3

configs/recipes/networks/mono_depth_res_net.yaml

@@ -0,0 +1,10 @@
+default:
+    file: depth/MonoDepthResNet
+    encoder:
+        version: 18
+        pretrained: True
+        num_rgb_in: 1
+    decoder:
+        use_skips: True
+        activation: sigmoid
+        num_ch_out: 1

configs/recipes/networks/multi_depth_res_net.yaml

@@ -0,0 +1,17 @@
+depthformer:
+    file: depth/MultiDepthResNet
+    encoder:
+        version: 18
+        pretrained: True
+        input_shape: [192,640]
+        adaptive_bins: True
+        depth_range: [0.1,100.]
+        depth_bin_range: [0.1,100.0]
+        depth_binning: sid
+        num_depth_bins: 128
+    decoder:
+        use_skips: True
+        use_aux_depth: False
+        activation: sigmoid
+        num_ch_out: 1
+    depth_range: [0.1,100.]

configs/recipes/networks/packnet.yaml

@@ -0,0 +1,10 @@
+default:
+    file: depth/PackNet
+    encoder:
+        version: 18
+        pretrained: True
+        num_rgb_in: 1
+    decoder:
+        use_skips: True
+        activation: sigmoid
+        num_ch_out: 1

configs/recipes/networks/pose_net.yaml

@@ -0,0 +1,5 @@
+default:
+    file: pose/PoseNet
+    version: 18
+    pretrained: True
+    num_rgb_in: 2

configs/recipes/networks/transformer.yaml

@@ -0,0 +1,12 @@
+depthformer:
+    file: transformers/MatchModule
+    context_adjustment:
+        expansion_ratio: 4
+        feat_dim: 16
+        num_blocks: 8
+    channel_dim: 128
+    nheads: 8
+    num_attn_layers: 6
+    min_depth: 0.1
+    max_depth: 100.0
+    num_bins: 128

configs/recipes/optimizers.yaml

@@ -0,0 +1,7 @@
+adam_20_05:
+    name: Adam
+    lr: 0.0001
+    scheduler:
+        name: StepLR
+        step_size: 20
+        gamma: 0.5

configs/recipes/save.yaml

@@ -0,0 +1,5 @@
+depth_splitname:
+    folder: /data/vidar/save
+    naming: splitname
+    rgb: [tgt]
+    depth: [viz,npz]

configs/recipes/wandb.yaml

@@ -0,0 +1,5 @@
+default:
+    folder: /data/vidar/wandb
+    entity: tri
+    project: vidar
+    num_validation_logs: 5

configs/recipes/wrapper.yaml

@@ -0,0 +1,10 @@
+default:
+    seed: 42
+    min_epochs: 0
+    max_epochs: 50
+    grad_scaler: False
+    find_unused_parameters: True
+    flip_lr_prob: 0.5
+    validate_first: True
+    validate_flipped: True
+    sync_batch_norm: True