adding files

.gitattributes

@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+figures/demo_image.gif filter=lfs diff=lfs merge=lfs -text
+figures/demo_type.png filter=lfs diff=lfs merge=lfs -text
+figures/demo_video.gif filter=lfs diff=lfs merge=lfs -text
+figures/demo_webapp.gif filter=lfs diff=lfs merge=lfs -text
+figures/fig_architecture.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_pyramid_blending.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_qualitative_dis.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_qualitative.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_qualitative2.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_qualitative3.jpg filter=lfs diff=lfs merge=lfs -text
+figures/fig_quantitative.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_quantitative2.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_quantitative3.png filter=lfs diff=lfs merge=lfs -text
+figures/fig_quantitative4.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+data/backbone_ckpt
+data/Train_Dataset
+data/Test_Dataset
+results/*
+snapshots
+snapshots/*
+Samples/*
+temp/*
+.idea
+.vscode

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

Expr.py

@@ -0,0 +1,14 @@
+import torch.cuda as cuda
+
+from utils.misc import *
+from run import *
+
+if __name__ == "__main__":
+    args = parse_args()
+    opt = load_config(args.config)
+    
+    train(opt, args)
+    cuda.empty_cache()
+    if args.local_rank <= 0:
+        test(opt, args)
+        evaluate(opt, args)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Taehun Kim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,186 @@
----
-license: afl-3.0
----
+# Revisiting Image Pyramid Structure for High Resolution Salient Object Detection (InSPyReNet)
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/salient-object-detection-on-hku-is)](https://paperswithcode.com/sota/salient-object-detection-on-hku-is?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/salient-object-detection-on-duts-te)](https://paperswithcode.com/sota/salient-object-detection-on-duts-te?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/salient-object-detection-on-pascal-s)](https://paperswithcode.com/sota/salient-object-detection-on-pascal-s?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/salient-object-detection-on-ecssd)](https://paperswithcode.com/sota/salient-object-detection-on-ecssd?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/salient-object-detection-on-dut-omron)](https://paperswithcode.com/sota/salient-object-detection-on-dut-omron?p=revisiting-image-pyramid-structure-for-high)
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/rgb-salient-object-detection-on-davis-s)](https://paperswithcode.com/sota/rgb-salient-object-detection-on-davis-s?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/rgb-salient-object-detection-on-hrsod)](https://paperswithcode.com/sota/rgb-salient-object-detection-on-hrsod?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/rgb-salient-object-detection-on-uhrsd)](https://paperswithcode.com/sota/rgb-salient-object-detection-on-uhrsd?p=revisiting-image-pyramid-structure-for-high)
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/dichotomous-image-segmentation-on-dis-vd)](https://paperswithcode.com/sota/dichotomous-image-segmentation-on-dis-vd?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/dichotomous-image-segmentation-on-dis-te1)](https://paperswithcode.com/sota/dichotomous-image-segmentation-on-dis-te1?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/dichotomous-image-segmentation-on-dis-te2)](https://paperswithcode.com/sota/dichotomous-image-segmentation-on-dis-te2?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/dichotomous-image-segmentation-on-dis-te3)](https://paperswithcode.com/sota/dichotomous-image-segmentation-on-dis-te3?p=revisiting-image-pyramid-structure-for-high)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/revisiting-image-pyramid-structure-for-high/dichotomous-image-segmentation-on-dis-te4)](https://paperswithcode.com/sota/dichotomous-image-segmentation-on-dis-te4?p=revisiting-image-pyramid-structure-for-high)
+
+Official PyTorch implementation of PyTorch implementation of Revisiting Image Pyramid Structure for High Resolution Salient Object Detection (InSPyReNet)
+
+To appear in the 16th Asian Conference on Computer Vision (ACCV2022)
+
+<p align="center">
+<a href="https://github.com/plemeri/InSPyReNet/blob/main/LICENSE"><img  src="https://img.shields.io/badge/license-MIT-blue"></a>
+<a href="https://arxiv.org/abs/2209.09475"><img  src="https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg" ></a>
+<a href="https://openaccess.thecvf.com/content/ACCV2022/html/Kim_Revisiting_Image_Pyramid_Structure_for_High_Resolution_Salient_Object_Detection_ACCV_2022_paper.html"><img  src="https://img.shields.io/static/v1?label=inproceedings&message=Paper&color=orange"></a>
+<a href="https://huggingface.co/spaces/taskswithcode/salient-object-detection"><img  src="https://img.shields.io/static/v1?label=HuggingFace&message=Demo&color=yellow"></a>
+<a href="https://www.taskswithcode.com/salient_object_detection/"><img  src="https://img.shields.io/static/v1?label=TasksWithCode&message=Demo&color=blue"></a>
+<a href="https://colab.research.google.com/github/taskswithcode/InSPyReNet/blob/main/TWCSOD.ipynb"><img  src="https://colab.research.google.com/assets/colab-badge.svg"></a>
+</p>
+
+> [Taehun Kim](https://scholar.google.co.kr/citations?user=f12-9yQAAAAJ&hl=en), [Kunhee Kim](https://scholar.google.co.kr/citations?user=6sU5r7MAAAAJ&hl=en), [Joonyeong Lee](https://scholar.google.co.kr/citations?hl=en&user=pOM4zSYAAAAJ), Dongmin Cha, [Jiho Lee](https://scholar.google.co.kr/citations?user=1Q1awj8AAAAJ&hl=en), [Daijin Kim](https://scholar.google.co.kr/citations?user=Mw6anjAAAAAJ&hl=en)
+
+> **Abstract:**
+  Salient object detection (SOD) has been in the spotlight recently, yet has been studied less for high-resolution (HR) images. 
+  Unfortunately, HR images and their pixel-level annotations are certainly more labor-intensive and time-consuming compared to low-resolution (LR) images.
+  Therefore, we propose an image pyramid-based SOD framework, Inverse Saliency Pyramid Reconstruction Network (InSPyReNet), for HR prediction without any of HR datasets.
+  We design InSPyReNet to produce a strict image pyramid structure of saliency map, which enables to ensemble multiple results with pyramid-based image blending.
+  For HR prediction, we design a pyramid blending method which synthesizes two different image pyramids from a pair of LR and HR scale from the same image to overcome effective receptive field (ERF) discrepancy. Our extensive evaluation on public LR and HR SOD benchmarks demonstrates that InSPyReNet surpasses the State-of-the-Art (SotA) methods on various SOD metrics and boundary accuracy.
+
+## Contents
+
+1. [News](#news-newspaper)
+2. [Demo](#demo-rocket)
+3. [Applications](#applications-video_game)
+4. [Easy Download](#easy-download-cake)
+5. [Getting Started](#getting-started-flight_departure)
+6. [Model Zoo](#model-zoo-giraffe)
+7. [Results](#results-100)
+    * [Quantitative Results](#quantitative-results)
+    * [Qualitative Results](#qualitative-results)
+8. [Citation](#citation)
+9. [Acknowledgement](#acknowledgement)
+    * [Special Thanks to](#special-thanks-to-tada)
+10. [References](#references)
+  
+## News :newspaper:
+
+[2022.10.04] [TasksWithCode](https://github.com/taskswithcode) mentioned our work in [Blog](https://medium.com/@taskswithcode/twc-9-7c960c921f69) and reproducing our work on [Colab](https://github.com/taskswithcode/InSPyReNet). Thank you for your attention!
+
+[2022.10.20] We trained our model on [Dichotomous Image Segmentation dataset (DIS5K)](https://xuebinqin.github.io/dis/index.html) and showed competitive results! Trained checkpoint and pre-computed segmentation masks are available in [Model Zoo](./docs/model_zoo.md)). Also, you can check our qualitative and quantitative results in [Results](#results-100) section.
+
+[2022.10.28] Multi GPU training for latest pytorch is now available.
+
+[2022.10.31] [TasksWithCode](https://github.com/taskswithcode) provided an amazing web demo with [HuggingFace](https://huggingface.co). Visit the [WepApp](https://huggingface.co/spaces/taskswithcode/salient-object-detection) and try with your image! 
+
+[2022.11.09] :car: Lane segmentation for driving scene built based on InSPyReNet is available in [`LaneSOD`](https://github.com/plemeri/LaneSOD) repository.
+
+[2022.11.18] I am speaking at The 16th Asian Conference on Computer Vision (ACCV2022). Please check out my talk if you're attending the event! #ACCV2022 #Macau - via #Whova event app
+
+[2022.11.23] We made our work available on pypi package. Please visit [`transparent-background`](https://github.com/plemeri/transparent-background) to download our tool and try on your machine. It works as command-line tool and python API.
+
+[2023.01.18] [rsreetech](https://github.com/rsreetech) shared a tutorial for our pypi package [`transparent-background`](https://github.com/plemeri/transparent-background) using colab. :tv: [[Youtube](https://www.youtube.com/watch?v=jKuQEnKmv4A)]
+
+## Demo :rocket:
+
+[Image Sample](./figures/demo_image.gif) | [Video Sample](./figures/demo_video.gif)
+:-:|:-:
+<img src=./figures/demo_image.gif height=200px> | <img src=./figures/demo_video.gif height=200px>
+
+## Applications :video_game: 
+Here are some applications/extensions of our work.
+### Web Application <img src=https://huggingface.co/front/assets/huggingface_logo-noborder.svg height="20px" width="20px"> 
+[TasksWithCode](https://github.com/taskswithcode) provided [WepApp](https://huggingface.co/spaces/taskswithcode/salient-object-detection) on HuggingFace to generate your own results!
+
+[Web Demo](https://huggingface.co/spaces/taskswithcode/salient-object-detection) |
+|:-:
+<img src=./figures/demo_webapp.gif height=200px> |
+
+### Command-line Tool / Python API :pager: 
+Try using our model as command-line tool or python API. More details about how to use is available on [`transparent-background`](https://github.com/plemeri/transparent-background).
+```bash
+pip install transparent-background
+```
+### Lane Segmentation :car: 
+We extend our model to detect lane markers in a driving scene in [`LaneSOD`](https://github.com/plemeri/LaneSOD)
+
+[Lane Segmentation](https://github.com/plemeri/LaneSOD) |
+|:-:
+<img src=https://github.com/plemeri/LaneSOD/blob/main/figures/Teaser.gif height=200px> |
+
+## Easy Download :cake: 
+
+<details><summary>How to use easy download</summary>
+<p>
+
+Downloading each dataset, checkpoint is quite bothering, even for me :zzz:. Instead, you can download data we provide including `ImageNet pre-trained backbone checkpoints`, `Training Datasets`, `Testing Datasets for benchmark`, `Pre-trained model checkpoints`, `Pre-computed saliency maps` with single command below.
+```bash
+python utils/download.py --extra --dest [DEST]
+```
+
+* `--extra, -e`: Without this argument, only the datasets, checkpoint, and results from our main paper will be downloaded. Otherwise, all data will be downloaded including results from supplementary material and DIS5K results.
+* `--dest [DEST], -d [DEST]`: If you want to specify the destination, use this argument. It will automatically create a symbolic links of the destination folders inside `data` and `snapshots`. Use this argument if you want to download data on other physical disk. Otherwise, it will download inside this repository folder.
+
+If you want to download a certain checkpoint or pre-computed map, please refer to [Getting Started](#getting-started-flight_departure) and [Model Zoo](#model-zoo-giraffe).
+
+</p>
+</details>
+
+## Getting Started :flight_departure:
+
+Please refer to [getting_started.md](./docs/getting_started.md) for training, testing, and evaluating on benchmarks, and inferencing on your own images.
+
+## Model Zoo :giraffe:
+
+Please refer to [model_zoo.md](./docs/model_zoo.md) for downloading pre-trained models and pre-computed saliency maps.
+
+## Results :100:
+
+### Quantitative Results
+
+[LR Benchmark](./figures/fig_quantitative.png) | [HR Benchmark](./figures/fig_quantitative2.png) | [HR Benchmark (Trained with extra DB)](./figures/fig_quantitative3.png) | [DIS](./figures/fig_quantitative4.png)
+:-:|:-:|:-:|:-:
+<img src=./figures/fig_quantitative.png height=200px> | <img src=./figures/fig_quantitative2.png height=200px> | <img src=./figures/fig_quantitative3.png height=200px> | <img src=./figures/fig_quantitative4.png height=200px>
+
+</p>
+</details>
+
+### Qualitative Results
+
+[DAVIS-S & HRSOD](./figures/fig_qualitative.png) | [UHRSD](./figures/fig_qualitative2.png) | [UHRSD (w/ HR scale)](./figures/fig_qualitative3.jpg) | [DIS](./figures/fig_qualitative_dis.png)
+:-:|:-:|:-:|:-:
+<img src=./figures/fig_qualitative.png height=200px> | <img src=./figures/fig_qualitative2.png height=200px> | <img src=./figures/fig_qualitative3.jpg height=200px> | <img src=./figures/fig_qualitative_dis.png height=200px>
+
+## Citation
+
+```
+@inproceedings{kim2022revisiting,
+  title={Revisiting Image Pyramid Structure for High Resolution Salient Object Detection},
+  author={Kim, Taehun and Kim, Kunhee and Lee, Joonyeong and Cha, Dongmin and Lee, Jiho and Kim, Daijin},
+  booktitle={Proceedings of the Asian Conference on Computer Vision},
+  pages={108--124},
+  year={2022}
+}
+```
+
+## Acknowledgement
+
+This work was supported by Institute of Information & communications Technology Planning & Evaluation (IITP) grant funded by the Korea government (MSIT) 
+(No.2017-0-00897, Development of Object Detection and Recognition for Intelligent Vehicles) and 
+(No.B0101-15-0266, Development of High Performance Visual BigData Discovery Platform for Large-Scale Realtime Data Analysis)
+
+### Special Thanks to :tada:
+* [TasksWithCode](https://github.com/taskswithcode) team for sharing our work and making the most amazing web app demo.
+
+
+## References
+
+### Related Works
+
+* Towards High-Resolution Salient Object Detection ([paper](https://drive.google.com/open?id=15o-Fel0BSyNulGoptrxfHR0t22qMHlTr) | [github](https://github.com/yi94code/HRSOD))
+* Disentangled High Quality Salient Object Detection ([paper](https://openaccess.thecvf.com/content/ICCV2021/papers/Tang_Disentangled_High_Quality_Salient_Object_Detection_ICCV_2021_paper.pdf) | [github](https://github.com/luckybird1994/HQSOD))
+* Pyramid Grafting Network for One-Stage High Resolution Saliency Detection ([paper](https://openaccess.thecvf.com/content/CVPR2022/papers/Xie_Pyramid_Grafting_Network_for_One-Stage_High_Resolution_Saliency_Detection_CVPR_2022_paper.pdf) | [github](https://github.com/iCVTEAM/PGNet))
+
+### Resources
+
+* Backbones: [Res2Net](https://github.com/Res2Net/Res2Net-PretrainedModels), [Swin Transformer](https://github.com/microsoft/Swin-Transformer)
+
+* Datasets
+  * LR Benchmarks: [DUTS](http://saliencydetection.net/duts/), [DUT-OMRON](http://saliencydetection.net/dut-omron/), [ECSSD](http://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/dataset.html), [HKU-IS](https://i.cs.hku.hk/~gbli/deep_saliency.html), [PASCAL-S](http://cbi.gatech.edu/salobj/)
+  * HR Benchmarks: [DAVIS-S, HRSOD](https://github.com/yi94code/HRSOD), [UHRSD](https://github.com/iCVTEAM/PGNet)
+  * Dichotomous Image Segmentation: [DIS5K](https://xuebinqin.github.io/dis/index.html)
+
+* Evaluation Toolkit
+  * SOD Metrics (e.g., S-measure): [PySOD Metrics](https://github.com/lartpang/PySODMetrics)
+  * Boundary Metric (mBA): [CascadePSP](https://github.com/hkchengrex/CascadePSP)

configs/InSPyReNet_Res2Net50.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_Res2Net50"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: NULL
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DUTS-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: True
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_Res2Net50"
+    Debug:
+        keys: ['saliency', 'laplacian']
+
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S',  'DAVIS-S', 'HRSOD', 'UHRSD-TE']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_Res2Net50"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_Res2Net50"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S',  'DAVIS-S', 'HRSOD', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/InSPyReNet_SwinB.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DUTS-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_DH.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [1024, 1024]
+    threshold: NULL
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DUTS-TR', 'HRSOD-TR']
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 1
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DH"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DH"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_DH"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_DHU_LR.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DUTS-TR', 'HRSOD-TR', 'UHRSD-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DHU_LR"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DHU_LR"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_DHU_LR"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_DH_LR.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DUTS-TR', 'HRSOD-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DH_LR"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DH_LR"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_DH_LR"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_DIS5K.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [1024, 1024]
+    threshold: NULL
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DIS-TR']
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 1
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 180
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DIS5K"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets: ['DIS-VD', 'DIS-TE1', 'DIS-TE2', 'DIS-TE3', 'DIS-TE4']
+        transforms:
+            static_resize:
+                size: [1024, 1024]
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DIS5K"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_DIS5K"
+    result_path: "results"
+    datasets: ['DIS-VD', 'DIS-TE1', 'DIS-TE2', 'DIS-TE3', 'DIS-TE4']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_DIS5K_LR.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['DIS-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 180
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DIS5K_LR"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets: ['DIS-VD', 'DIS-TE1', 'DIS-TE2', 'DIS-TE3', 'DIS-TE4']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_DIS5K_LR"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_DIS5K_LR"
+    result_path: "results"
+    datasets: ['DIS-VD', 'DIS-TE1', 'DIS-TE2', 'DIS-TE3', 'DIS-TE4']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_HU.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [1024, 1024]
+    threshold: NULL
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['HRSOD-TR', 'UHRSD-TR']
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 1
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_HU"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_HU"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_HU"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/InSPyReNet_SwinB_HU_LR.yaml

@@ -0,0 +1,76 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Train_Dataset"
+        sets: ['HRSOD-TR', 'UHRSD-TR']
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_HU_LR"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/InSPyReNet_SwinB_HU_LR"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/InSPyReNet_SwinB_HU_LR"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/Plus_Ultra.yaml

@@ -0,0 +1,98 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [1024, 1024]
+    threshold: NULL
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data"
+        sets: [
+               'Train_Dataset/DUTS-TR', 
+               'Train_Dataset/HRSOD-TR', 
+               'Train_Dataset/UHRSD-TR',
+               'Train_Dataset/DIS-TR',
+               'Test_Dataset/DUTS-TE', 
+               'Test_Dataset/DUT-OMRON', 
+               'Test_Dataset/ECSSD', 
+               'Test_Dataset/HKU-IS', 
+               'Test_Dataset/PASCAL-S', 
+               'Test_Dataset/DAVIS-S', 
+               'Test_Dataset/HRSOD-TE',
+               'Test_Dataset/UHRSD-TE', 
+               'Test_Dataset/FSS-1000', 
+               'Test_Dataset/MSRA-10K',
+               'Test_Dataset/DIS-VD',
+               'Test_Dataset/DIS-TE1',
+               'Test_Dataset/DIS-TE2',
+               'Test_Dataset/DIS-TE3',
+               'Test_Dataset/DIS-TE4'
+               ]
+        transforms:
+            static_resize: 
+                size: [1024, 1024]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 1
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/Plus_Ultra"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            # static_resize: 
+            #     size: [1024, 1024]
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/Plus_Ultra"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/Plus_Ultra"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

configs/extra_dataset/Plus_Ultra_LR.yaml

@@ -0,0 +1,96 @@
+Model:
+    name: "InSPyReNet_SwinB"
+    depth: 64
+    pretrained: True
+    base_size: [384, 384]
+    threshold: 512
+
+Train:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data"
+        sets: [
+               'Train_Dataset/DUTS-TR', 
+               'Train_Dataset/HRSOD-TR', 
+               'Train_Dataset/UHRSD-TR',
+               'Train_Dataset/DIS-TR',
+               'Test_Dataset/DUTS-TE', 
+               'Test_Dataset/DUT-OMRON', 
+               'Test_Dataset/ECSSD', 
+               'Test_Dataset/HKU-IS', 
+               'Test_Dataset/PASCAL-S', 
+               'Test_Dataset/DAVIS-S', 
+               'Test_Dataset/HRSOD-TE',
+               'Test_Dataset/UHRSD-TE', 
+               'Test_Dataset/FSS-1000', 
+               'Test_Dataset/MSRA-10K',
+               'Test_Dataset/DIS-VD',
+               'Test_Dataset/DIS-TE1',
+               'Test_Dataset/DIS-TE2',
+               'Test_Dataset/DIS-TE3',
+               'Test_Dataset/DIS-TE4'
+               ]
+        transforms:
+            static_resize: 
+                size: [384, 384]
+            random_scale_crop:
+                range: [0.75, 1.25]
+            random_flip:
+                lr: True
+                ud: False
+            random_rotate:
+                range: [-10, 10]
+            random_image_enhance:
+                methods: ['contrast', 'sharpness', 'brightness']
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        batch_size: 6
+        shuffle: True
+        num_workers: 8
+        pin_memory: False
+    Optimizer:
+        type: "Adam"
+        lr: 1.0e-05
+        weight_decay: 0.0
+        mixed_precision: False
+    Scheduler:
+        type: "PolyLr"
+        epoch: 60
+        gamma: 0.9
+        minimum_lr: 1.0e-07
+        warmup_iteration: 12000
+    Checkpoint:
+        checkpoint_epoch: 1
+        checkpoint_dir: "snapshots/Plus_Ultra_LR"
+    Debug:
+        keys: ['saliency', 'laplacian']
+    
+Test:
+    Dataset:
+        type: "RGB_Dataset"
+        root: "data/Test_Dataset"
+        sets:  ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+        transforms:
+            dynamic_resize:
+                L: 1280
+            tonumpy: NULL
+            normalize: 
+                mean: [0.485, 0.456, 0.406]
+                std: [0.229, 0.224, 0.225]
+            totensor: NULL
+    Dataloader:
+        num_workers: 8
+        pin_memory: True
+    Checkpoint:
+        checkpoint_dir: "snapshots/Plus_Ultra_LR"
+
+Eval:
+    gt_root: "data/Test_Dataset"
+    pred_root: "snapshots/Plus_Ultra_LR"
+    result_path: "results"
+    datasets: ['DUTS-TE', 'DUT-OMRON', 'ECSSD', 'HKU-IS', 'PASCAL-S', 'DAVIS-S', 'HRSOD-TE', 'UHRSD-TE']
+    metrics: ['Sm', 'mae', 'adpEm', 'maxEm', 'avgEm', 'adpFm', 'maxFm', 'avgFm', 'wFm', 'mBA']

data/custom_transforms.py

@@ -0,0 +1,206 @@
+from email.mime import base
+import numpy as np
+from PIL import Image
+import os
+import sys
+import torch
+import torch.nn.functional as F
+from PIL import Image, ImageOps, ImageFilter, ImageEnhance
+from typing import Optional
+
+filepath = os.path.split(__file__)[0]
+repopath = os.path.split(filepath)[0]
+sys.path.append(repopath)
+
+from utils.misc import *
+        
+class static_resize:
+    # Resize for training
+    # size: h x w
+    def __init__(self, size=[384, 384], base_size=None):
+        self.size = size[::-1]
+        self.base_size = base_size[::-1] if base_size is not None else None
+            
+    def __call__(self, sample):
+        sample['image'] = sample['image'].resize(self.size, Image.BILINEAR)
+        if 'gt' in sample.keys():
+            sample['gt'] = sample['gt'].resize(self.size, Image.NEAREST)
+            
+        if self.base_size is not None:
+            sample['image_resized'] = sample['image'].resize(self.size, Image.BILINEAR)
+            if 'gt' in sample.keys():
+                sample['gt_resized'] = sample['gt'].resize(self.size, Image.NEAREST)
+                
+        return sample
+
+class dynamic_resize:
+    # base_size: h x w
+    def __init__(self, L=1280, base_size=[384, 384]): 
+        self.L = L
+        self.base_size = base_size[::-1]
+                    
+    def __call__(self, sample):
+        size = list(sample['image'].size)
+        if (size[0] >= size[1]) and size[1] > self.L: 
+            size[0] = size[0] / (size[1] / self.L)
+            size[1] = self.L
+        elif (size[1] > size[0]) and size[0] > self.L:
+            size[1] = size[1] / (size[0] / self.L)
+            size[0] = self.L
+        size = (int(round(size[0] / 32)) * 32, int(round(size[1] / 32)) * 32)
+    
+        if 'image' in sample.keys():
+            sample['image_resized'] = sample['image'].resize(self.base_size, Image.BILINEAR)
+            sample['image'] = sample['image'].resize(size, Image.BILINEAR)
+            
+        if 'gt' in sample.keys():
+            sample['gt_resized'] = sample['gt'].resize(self.base_size, Image.NEAREST)
+            sample['gt'] = sample['gt'].resize(size, Image.NEAREST)
+        
+        return sample
+        
+class random_scale_crop:
+    def __init__(self, range=[0.75, 1.25]):
+        self.range = range
+
+    def __call__(self, sample):
+        scale = np.random.random() * (self.range[1] - self.range[0]) + self.range[0]
+        if np.random.random() < 0.5:
+            for key in sample.keys():
+                if key in ['image', 'gt']:
+                    base_size = sample[key].size
+
+                    scale_size = tuple((np.array(base_size) * scale).round().astype(int))
+                    sample[key] = sample[key].resize(scale_size)
+
+                    lf = (sample[key].size[0] - base_size[0]) // 2
+                    up = (sample[key].size[1] - base_size[1]) // 2
+                    rg = (sample[key].size[0] + base_size[0]) // 2
+                    lw = (sample[key].size[1] + base_size[1]) // 2
+
+                    border = -min(0, min(lf, up))
+                    sample[key] = ImageOps.expand(sample[key], border=border)
+                    sample[key] = sample[key].crop((lf + border, up + border, rg + border, lw + border))
+        return sample
+
+class random_flip:
+    def __init__(self, lr=True, ud=True):
+        self.lr = lr
+        self.ud = ud
+
+    def __call__(self, sample):
+        lr = np.random.random() < 0.5 and self.lr is True
+        ud = np.random.random() < 0.5 and self.ud is True
+
+        for key in sample.keys():
+            if key in ['image', 'gt']:
+                sample[key] = np.array(sample[key])
+                if lr:
+                    sample[key] = np.fliplr(sample[key])
+                if ud:
+                    sample[key] = np.flipud(sample[key])
+                sample[key] = Image.fromarray(sample[key])
+
+        return sample
+
+class random_rotate:
+    def __init__(self, range=[0, 360], interval=1):
+        self.range = range
+        self.interval = interval
+
+    def __call__(self, sample):
+        rot = (np.random.randint(*self.range) // self.interval) * self.interval
+        rot = rot + 360 if rot < 0 else rot
+
+        if np.random.random() < 0.5:
+            for key in sample.keys():
+                if key in ['image', 'gt']:
+                    base_size = sample[key].size
+                    sample[key] = sample[key].rotate(rot, expand=True, fillcolor=255 if key == 'depth' else None)
+
+                    sample[key] = sample[key].crop(((sample[key].size[0] - base_size[0]) // 2,
+                                                    (sample[key].size[1] - base_size[1]) // 2,
+                                                    (sample[key].size[0] + base_size[0]) // 2,
+                                                    (sample[key].size[1] + base_size[1]) // 2))
+
+        return sample
+
+class random_image_enhance:
+    def __init__(self, methods=['contrast', 'brightness', 'sharpness']):
+        self.enhance_method = []
+        if 'contrast' in methods:
+            self.enhance_method.append(ImageEnhance.Contrast)
+        if 'brightness' in methods:
+            self.enhance_method.append(ImageEnhance.Brightness)
+        if 'sharpness' in methods:
+            self.enhance_method.append(ImageEnhance.Sharpness)
+
+    def __call__(self, sample):
+        if 'image' in sample.keys():
+            np.random.shuffle(self.enhance_method)
+
+            for method in self.enhance_method:
+                if np.random.random() > 0.5:
+                    enhancer = method(sample['image'])
+                    factor = float(1 + np.random.random() / 10)
+                    sample['image'] = enhancer.enhance(factor)
+
+        return sample
+
+class tonumpy:
+    def __init__(self):
+        pass
+
+    def __call__(self, sample):
+        for key in sample.keys():
+            if key in ['image', 'image_resized', 'gt', 'gt_resized']:
+                sample[key] = np.array(sample[key], dtype=np.float32)
+
+        return sample
+
+class normalize:
+    def __init__(self, mean: Optional[list]=None, std: Optional[list]=None, div=255):
+        self.mean = mean if mean is not None else 0.0
+        self.std = std if std is not None else 1.0
+        self.div = div
+        
+    def __call__(self, sample):
+        if 'image' in sample.keys():
+            sample['image'] /= self.div
+            sample['image'] -= self.mean
+            sample['image'] /= self.std
+            
+        if 'image_resized' in sample.keys():
+            sample['image_resized'] /= self.div
+            sample['image_resized'] -= self.mean
+            sample['image_resized'] /= self.std
+
+        if 'gt' in sample.keys():
+            sample['gt'] /= self.div
+            
+        if 'gt_resized' in sample.keys():
+            sample['gt_resized'] /= self.div
+
+        return sample
+class totensor:
+    def __init__(self):
+        pass
+
+    def __call__(self, sample):
+        if 'image' in sample.keys():
+            sample['image'] = sample['image'].transpose((2, 0, 1))
+            sample['image'] = torch.from_numpy(sample['image']).float()
+        
+        if 'image_resized' in sample.keys():
+            sample['image_resized'] = sample['image_resized'].transpose((2, 0, 1))
+            sample['image_resized'] = torch.from_numpy(sample['image_resized']).float()
+            
+        if 'gt' in sample.keys():
+            sample['gt'] = torch.from_numpy(sample['gt'])
+            sample['gt'] = sample['gt'].unsqueeze(dim=0)
+            
+        if 'gt_resized' in sample.keys():
+            sample['gt_resized'] = torch.from_numpy(sample['gt_resized'])
+            sample['gt_resized'] = sample['gt_resized'].unsqueeze(dim=0)
+
+        return sample

data/dataloader.py

@@ -0,0 +1,243 @@
+import os
+import cv2
+import sys
+
+import numpy as np
+import torchvision.transforms as transforms
+
+from torch.utils.data.dataset import Dataset
+from PIL import Image
+from threading import Thread
+
+filepath = os.path.split(__file__)[0]
+repopath = os.path.split(filepath)[0]
+sys.path.append(repopath)
+
+from data.custom_transforms import *
+from utils.misc import *
+
+Image.MAX_IMAGE_PIXELS = None
+
+def get_transform(tfs):
+    comp = []
+    for key, value in zip(tfs.keys(), tfs.values()):
+        if value is not None:
+            tf = eval(key)(**value)
+        else:
+            tf = eval(key)()
+        comp.append(tf)
+    return transforms.Compose(comp)
+
+class RGB_Dataset(Dataset):
+    def __init__(self, root, sets, tfs):
+        self.images, self.gts = [], []
+        
+        for set in sets:
+            image_root, gt_root = os.path.join(root, set, 'images'), os.path.join(root, set, 'masks')
+
+            images = [os.path.join(image_root, f) for f in os.listdir(image_root) if f.lower().endswith(('.jpg', '.png'))]
+            images = sort(images)
+            
+            gts = [os.path.join(gt_root, f) for f in os.listdir(gt_root) if f.lower().endswith(('.jpg', '.png'))]
+            gts = sort(gts)
+            
+            self.images.extend(images)
+            self.gts.extend(gts)
+        
+        self.filter_files()
+        
+        self.size = len(self.images)
+        self.transform = get_transform(tfs)
+        
+    def __getitem__(self, index):
+        image = Image.open(self.images[index]).convert('RGB')
+        gt = Image.open(self.gts[index]).convert('L')
+        shape = gt.size[::-1]
+        name = self.images[index].split(os.sep)[-1]
+        name = os.path.splitext(name)[0]
+            
+        sample = {'image': image, 'gt': gt, 'name': name, 'shape': shape}
+
+        sample = self.transform(sample)
+        return sample
+
+    def filter_files(self):
+        assert len(self.images) == len(self.gts)
+        images, gts = [], []
+        for img_path, gt_path in zip(self.images, self.gts):
+            img, gt = Image.open(img_path), Image.open(gt_path)
+            if img.size == gt.size:
+                images.append(img_path)
+                gts.append(gt_path)
+        self.images, self.gts = images, gts
+
+    def __len__(self):
+        return self.size
+    
+class ImageLoader:
+    def __init__(self, root, tfs):
+        if os.path.isdir(root):
+            self.images = [os.path.join(root, f) for f in os.listdir(root) if f.lower().endswith(('.jpg', '.png', '.jpeg'))]
+            self.images = sort(self.images)
+        elif os.path.isfile(root):
+            self.images = [root]
+        self.size = len(self.images)
+        self.transform = get_transform(tfs)
+
+    def __iter__(self):
+        self.index = 0
+        return self
+
+    def __next__(self):
+        if self.index == self.size:
+            raise StopIteration
+        image = Image.open(self.images[self.index]).convert('RGB')
+        shape = image.size[::-1]
+        name = self.images[self.index].split(os.sep)[-1]
+        name = os.path.splitext(name)[0]
+            
+        sample = {'image': image, 'name': name, 'shape': shape, 'original': image}
+        sample = self.transform(sample)
+        sample['image'] = sample['image'].unsqueeze(0)
+        if 'image_resized' in sample.keys():
+            sample['image_resized'] = sample['image_resized'].unsqueeze(0)
+        
+        self.index += 1
+        return sample
+
+    def __len__(self):
+        return self.size
+    
+class VideoLoader:
+    def __init__(self, root, tfs):
+        if os.path.isdir(root):
+            self.videos = [os.path.join(root, f) for f in os.listdir(root) if f.lower().endswith(('.mp4', '.avi', 'mov'))]
+        elif os.path.isfile(root):
+            self.videos = [root]
+        self.size = len(self.videos)
+        self.transform = get_transform(tfs)
+
+    def __iter__(self):
+        self.index = 0
+        self.cap = None
+        self.fps = None
+        return self
+
+    def __next__(self):
+        if self.index == self.size:
+            raise StopIteration
+        
+        if self.cap is None:
+            self.cap = cv2.VideoCapture(self.videos[self.index])
+            self.fps = self.cap.get(cv2.CAP_PROP_FPS)
+        ret, frame = self.cap.read()
+        name = self.videos[self.index].split(os.sep)[-1]
+        name = os.path.splitext(name)[0]
+        if ret is False:
+            self.cap.release()
+            self.cap = None
+            sample = {'image': None, 'shape': None, 'name': name, 'original': None}
+            self.index += 1
+        
+        else:
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            image = Image.fromarray(frame).convert('RGB')
+            shape = image.size[::-1]
+            sample = {'image': image, 'shape': shape, 'name': name, 'original': image}
+            sample = self.transform(sample)
+            sample['image'] = sample['image'].unsqueeze(0)
+            if 'image_resized' in sample.keys():
+                sample['image_resized'] = sample['image_resized'].unsqueeze(0)
+            
+        return sample
+    
+    def __len__(self):
+        return self.size
+    
+
+class WebcamLoader:
+    def __init__(self, ID, tfs):
+        self.ID = int(ID)
+        self.cap = cv2.VideoCapture(self.ID)
+        self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+        self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+        self.transform = get_transform(tfs)
+        self.imgs = []
+        self.imgs.append(self.cap.read()[1])
+        self.thread = Thread(target=self.update, daemon=True)
+        self.thread.start()
+        
+    def update(self):
+        while self.cap.isOpened():
+            ret, frame = self.cap.read()
+            if ret is True:
+                self.imgs.append(frame)
+            else:
+                break
+        
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if len(self.imgs) > 0:
+            frame = self.imgs[-1]
+        else:
+            frame = np.zeros((480, 640, 3)).astype(np.uint8)
+        if self.thread.is_alive() is False or cv2.waitKey(1) == ord('q'):
+            cv2.destroyAllWindows()
+            raise StopIteration
+        
+        else:
+            image = Image.fromarray(frame).convert('RGB')
+            shape = image.size[::-1]
+            sample = {'image': image, 'shape': shape, 'name': 'webcam', 'original': image}
+            sample = self.transform(sample)
+            sample['image'] = sample['image'].unsqueeze(0)
+            if 'image_resized' in sample.keys():
+                sample['image_resized'] = sample['image_resized'].unsqueeze(0)
+        
+        del self.imgs[:-1]
+        return sample
+
+
+    def __len__(self):
+        return 0
+    
+class RefinementLoader:
+    def __init__(self, image_dir, seg_dir, tfs):
+        self.images = [os.path.join(image_dir, f) for f in os.listdir(image_dir) if f.lower().endswith(('.jpg', '.png', '.jpeg'))]
+        self.images = sort(self.images)
+        
+        self.segs = [os.path.join(seg_dir, f) for f in os.listdir(seg_dir) if f.lower().endswith(('.jpg', '.png', '.jpeg'))]
+        self.segs = sort(self.segs)
+            
+        self.size = len(self.images)
+        self.transform = get_transform(tfs)
+
+    def __iter__(self):
+        self.index = 0
+        return self
+
+    def __next__(self):
+        if self.index == self.size:
+            raise StopIteration
+        image = Image.open(self.images[self.index]).convert('RGB')
+        seg = Image.open(self.segs[self.index]).convert('L')
+        shape = image.size[::-1]
+        name = self.images[self.index].split(os.sep)[-1]
+        name = os.path.splitext(name)[0]
+            
+        sample = {'image': image, 'gt': seg, 'name': name, 'shape': shape, 'original': image}
+        sample = self.transform(sample)
+        sample['image'] = sample['image'].unsqueeze(0)
+        sample['mask'] = sample['gt'].unsqueeze(0)
+        if 'image_resized' in sample.keys():
+            sample['image_resized'] = sample['image_resized'].unsqueeze(0)
+        del sample['gt']
+        
+        self.index += 1
+        return sample
+
+    def __len__(self):
+        return self.size
+    

docs/getting_started.md

@@ -0,0 +1,138 @@
+# Getting Started :flight_departure:
+
+## Create environment
+  * Create conda environment with following command `conda create -y -n inspyrenet python`
+  * Activate environment with following command `conda activate inspyrenet`
+  * Install latest PyTorch from [Official Website](https://pytorch.org/get-started/locally/)
+    * Linux [2022.11.09]
+    ```
+    pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
+    ```
+  * Install requirements with following command `pip install -r requirements.txt`
+  
+## Preparation
+
+* For training, you may need training datasets and ImageNet pre-trained checkpoints for the backbone. For testing (inference), you may need test datasets (sample images).
+* Training datasets are expected to be located under [Train.Dataset.root](https://github.com/plemeri/InSPyReNet/blob/main/configs/InSPyReNet_SwinB.yaml#L11). Likewise, testing datasets should be under [Test.Dataset.root](https://github.com/plemeri/InSPyReNet/blob/main/configs/InSPyReNet_SwinB.yaml#L55).
+* Each dataset folder should contain `images` folder and `masks` folder for images and ground truth masks respectively.
+* You may use multiple training datasets by listing dataset folders for [Train.Dataset.sets](https://github.com/plemeri/InSPyReNet/blob/main/configs/InSPyReNet_SwinB.yaml#L12), such as `[DUTS-TR] -> [DUTS-TR, HRSOD-TR, UHRSD-TR]`.
+
+### Backbone Checkpoints
+Item | Destination Folder | OneDrive | GDrive
+:-|:-|:-|:-
+Res2Net50 checkpoint | `data/backbone_ckpt/*.pth` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUO7GDBwoC9CulTPdnq_yhQBlc0SIyyELMy3OmrNhOjcGg?e=T3PVyG&download=1) | [Link](https://drive.google.com/file/d/1MMhioAsZ-oYa5FpnTi22XBGh5HkjLX3y/view?usp=sharing)
+SwinB checkpoint     | `data/backbone_ckpt/*.pth` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESlYCLy0endMhcZm9eC2A4ABatxupp4UPh03EcqFjbtSRw?e=7y6lLt&download=1) | [Link](https://drive.google.com/file/d/1fBJFMupe5pV-Vtou-k8LTvHclWs0y1bI/view?usp=sharing)
+
+* We changed Res2Net50 checkpoint to resolve an error while training with DDP. Please refer to [issue #9](https://github.com/plemeri/InSPyReNet/issues/9).
+
+### Train Datasets
+Item | Destination Folder | OneDrive | GDrive
+:-|:-|:-|:-
+DUTS-TR | `data/Train_Dataset/...`   | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EQ7L2XS-5YFMuJGee7o7HQ8BdRSLO8utbC_zRrv-KtqQ3Q?e=bCSIeo&download=1) | [Link](https://drive.google.com/file/d/1hy5UTq65uQWFO5yzhEn9KFIbdvhduThP/view?usp=share_link)
+
+### Extra Train Datasets (High Resolution, Optional)
+Item | Destination Folder | OneDrive | GDrive
+:-|:-|:-|:-
+HRSOD-TR | `data/Train_Dataset/...`   | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfUx92hUgZJNrWPj46PC0yEBXcorQskXOCSz8SnGH5AcLQ?e=WA5pc6&download=1) | N/A
+UHRSD-TR | `data/Train_Dataset/...`   | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ea4_UCbsKmhKnMCccAJOTLgBmQFsQ4KhJSf2jx8WQqj3Wg?e=18kYZS&download=1) | N/A
+DIS-TR   | `data/Train_Dataset/...`   | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZtZJ493tVNJjBIpNLdus68B3u906PdWtHsf87pulj78jw?e=bUg2UQ&download=1) | N/A
+
+### Test Datasets
+Item | Destination Folder | OneDrive | GDrive
+:-|:-|:-|:-
+DUTS-TE   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfuCxjveXphPpIska9BxHDMBHpYroEKdVlq9HsonZ4wLDw?e=Mz5giA&download=1) | [Link](https://drive.google.com/file/d/1w4pigcQe9zplMulp1rAwmp6yYXmEbmvy/view?usp=share_link) 
+DUT-OMRON | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ERvApm9rHH5LiR4NJoWHqDoBCneUQNextk8EjQ_Hy0bUHg?e=wTRZQb&download=1) | [Link](https://drive.google.com/file/d/1qIm_GQLLQkP6s-xDZhmp_FEAalavJDXf/view?usp=sharing) 
+ECSSD     | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ES_GCdS0yblBmnRaDZ8xmKQBPU_qeECTVB9vlPUups8bnA?e=POVAlG&download=1) | [Link](https://drive.google.com/file/d/1qk_12KLGX6FPr1P_S9dQ7vXKaMqyIRoA/view?usp=sharing) 
+HKU-IS    | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EYBRVvC1MJRAgSfzt0zaG94BU_UWaVrvpv4tjogu4vSV6w?e=TKN7hQ&download=1) | [Link](https://drive.google.com/file/d/1H3szJYbr5_CRCzrYfhPHThTgszkKd1EU/view?usp=share_link) 
+PASCAL-S  | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfUDGDckMnZHhEPy8YQGwBQB5MN3qInBkEygpIr7ccJdTQ?e=YarZaQ&download=1) | [Link](https://drive.google.com/file/d/1h0IE2DlUt0HHZcvzMV5FCxtZqQqh9Ztf/view?usp=sharing)
+DAVIS-S   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ebam8I2o-tRJgADcq-r9YOkBCDyaAdWBVWyfN-xCYyAfDQ?e=Mqz8cK&download=1) | [Link](https://drive.google.com/file/d/15F0dy9o02LPTlpUbnD9NJlGeKyKU3zOz/view?usp=sharing)
+HRSOD-TE  | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EbHOQZKC59xIpIdrM11ulWsBHRYY1wZY2njjWCDFXvT6IA?e=wls17m&download=1) | [Link](https://drive.google.com/file/d/1KnUCsvluS4kP2HwUFVRbKU8RK_v6rv2N/view?usp=sharing)
+UHRSD-TE  | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUpc8QJffNpNpESv-vpBi40BppucqOoXm_IaK7HYJkuOog?e=JTjGmS&download=1) | [Link](https://drive.google.com/file/d/1niiHBo9LX6-I3KsEWYOi_s6cul80IYvK/view?usp=sharing)
+
+### Extra Test Datasets (Optional) 
+Item | Destination Folder | OneDrive | GDrive
+:-|:-|:-|:-
+FSS-1000  | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EaP6DogjMAVCtTKcC_Bx-YoBoBSWBo90lesVcMyuCN35NA?e=0DDohA&download=1) | N/A
+MSRA-10K  | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EauXsIkxBzhDjio6fW0TubUB4L7YJc0GMTaq7VfjI2nPsg?e=c5DIxg&download=1) | N/A
+DIS-VD    | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EYJm3BqheaxNhdVoMt6X41gBgVnE4dulBwkp6pbOQtcIrQ?e=T6dtXm&download=1) | [Link](https://drive.google.com/file/d/1jhlZb3QyNPkc0o8nL3RWF0MLuVsVtJju/view?usp=sharing)
+DIS-TE1   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EcGYE_Gc0cVHoHi_qUtmsawB_5v9RSpJS5PIAPlLu6xo9A?e=Nu5mkQ&download=1) | [Link](https://drive.google.com/file/d/1iz8Y4uaX3ZBy42N2MIOkmNb0D5jroFPJ/view?usp=sharing)
+DIS-TE2   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EdhgMdbZ049GvMv7tNrjbbQB1wL9Ok85YshiXIkgLyTfkQ?e=mPA6Po&download=1) | [Link](https://drive.google.com/file/d/1DWSoWogTWDuS2PFbD1Qx9P8_SnSv2zTe/view?usp=sharing)
+DIS-TE3   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EcxXYC_3rXxKsQBrp6BdNb4BOKxBK3_vsR9RL76n7YVG-g?e=2M0cse&download=1) | [Link](https://drive.google.com/file/d/1bIVSjsxCjMrcmV1fsGplkKl9ORiiiJTZ/view?usp=sharing)
+DIS-TE4   | `data/Test_Dataset/...` | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EdkG2SUi8flJvoYbHHOmvMABsGhkCJCsLLZlaV2E_SZimA?e=zlM2kC&download=1) | [Link](https://drive.google.com/file/d/1VuPNqkGTP1H4BFEHe807dTIkv8Kfzk5_/view?usp=sharing)
+
+## Train & Evaluate
+
+  * Train InSPyReNet
+  ```
+  # Single GPU
+  python run/Train.py --config configs/InSPyReNet_SwinB.yaml --verbose
+  
+  # Multi GPUs with DDP (e.g., 4 GPUs)
+  torchrun --standalone --nproc_per_node=4 run/Train.py --config configs/InSPyReNet_SwinB.yaml --verbose
+
+  # Multi GPUs with DDP with designated devices (e.g., 2 GPUs - 0 and 1)
+  CUDA_VISIBLE_DEVICES=0,1 torchrun --standalone --nproc_per_node=2 run/Train.py --config configs/InSPyReNet_SwinB.yaml --verbose
+  ```
+
+  * `--config [CONFIG_FILE], -c [CONFIG_FILE]`: config file path for training.
+  * `--resume, -r`: use this argument to resume from last checkpoint.
+  * `--verbose, -v`: use this argument to output progress info.
+  * `--debug, -d`: use this argument to save debug images every epoch.
+
+  * Train with extra training datasets can be done by just changing [Train.Dataset.sets](https://github.com/plemeri/InSPyReNet/blob/main/configs/InSPyReNet_SwinB.yaml#L12) in the `yaml` config file, which is just simply adding more directories (e.g., HRSOD-TR, HRSOD-TR, UHRSD-TR, ...):
+   ```
+   Train:
+     Dataset:
+         type: "RGB_Dataset"
+         root: "data/RGB_Dataset/Train_Dataset"
+         sets: ['DUTS-TR'] --> ['DUTS-TR', 'HRSOD-TR', 'UHRSD-TR']
+   ```
+  * Inference for test benchmarks
+  ```
+  python run/Test.py --config configs/InSPyReNet_SwinB.yaml --verbose
+  ```
+  * Evaluate metrics
+  ```
+  python run/Eval.py --config configs/InSPyReNet_SwinB.yaml --verbose
+  ```
+
+  * All-in-One command (Train, Test, Eval in single command)
+  ```
+  # Single GPU
+  python Expr.py --config configs/InSPyReNet_SwinB.yaml --verbose
+
+  # Multi GPUs with DDP (e.g., 4 GPUs)
+  torchrun --standalone --nproc_per_node=4 Expr.py --config configs/InSPyReNet_SwinB.yaml --verbose
+
+  # Multi GPUs with DDP with designated devices (e.g., 2 GPUs - 0 and 1)
+  CUDA_VISIBLE_DEVICES=0,1 torchrun --standalone --nproc_per_node=2 Expr.py --config configs/InSPyReNet_SwinB.yaml --verbose
+  ```
+## Inference on your own data
+  * You can inference your own single image or images (.jpg, .jpeg, and .png are supported), single video or videos (.mp4, .mov, and .avi are supported), and webcam input (ubuntu and macos are tested so far).
+  ```
+  python run/Inference.py --config configs/InSPyReNet_SwinB.yaml --source [SOURCE] --dest [DEST] --type [TYPE] --gpu --jit --verbose
+  ```
+
+  * `--source [SOURCE]`: Specify your data in this argument.
+    * Single image - `image.png`
+    * Folder containing images - `path/to/img/folder`
+    * Single video - `video.mp4`
+    * Folder containing videos - `path/to/vid/folder`
+    * Webcam input: `0` (may vary depends on your device.)
+  * `--dest [DEST]` (optional): Specify your destination folder. If not specified, it will be saved in `results` folder.
+  * `--type [TYPE]`: Choose between `map` `green`, `rgba`, `blur`, `overlay`, and another image file.
+    * `map` will output saliency map only. 
+    * `green` will change the background with green screen. 
+    * `rgba` will generate RGBA output regarding saliency score as an alpha map. Note that this will not work for video and webcam input. 
+    * `blur` will blur the background.
+    * `overlay` will cover the salient object with translucent green color, and highlight the edges.
+    * Another image file (e.g., `backgroud.png`) will be used as a background, and the object will be overlapped on it.
+    <details><summary>Examples of TYPE argument</summary>
+    <p>
+    <img src=../figures/demo_type.png >
+    </p>
+    </details>
+  * `--gpu`: Use this argument if you want to use GPU. 
+  * `--jit`: Slightly improves inference speed when used. 
+  * `--verbose`: Use when you want to visualize progress.
+

docs/model_zoo.md

@@ -0,0 +1,65 @@
+# Model Zoo :giraffe:
+
+## Pre-trained Checkpoints
+
+:label: Note: If you want to try our trained checkpoints below, please make sure to locate `latest.pth` file to the [Test.Checkpoint.checkpoint_dir](https://github.com/plemeri/InSPyReNet/blob/main/configs/InSPyReNet_SwinB.yaml#L69). 
+
+### Trained with LR dataset only (DUTS-TR, 384 X 384)
+
+Backbone |  Train DB  | Config | OneDrive | GDrive
+:-|:-|:-|:-|:-
+Res2Net50 | DUTS-TR | [InSPyReNet_Res2Net50](../configs/InSPyReNet_Res2Net50.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ERqm7RPeNBFPvVxkA5P5G2AB-mtFsiYkCNHnBf0DcwpFzw?e=nayVno&download=1) | [Link](https://drive.google.com/file/d/12moRuU8F0-xRvE16bVg6mkGWDuqYHJor/view?usp=sharing)
+SwinB | DUTS-TR | [InSPyReNet_SwinB](../configs/InSPyReNet_SwinB.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EV0ow4E8LddCgu5tAuAkMbcBpBYoEDmJgQg5wkiuvLoQUA?e=cOZspv&download=1) | [Link](https://drive.google.com/file/d/1k5hNJImgEgSmz-ZeJEEb_dVkrOnswVMq/view?usp=sharing)
+
+### Trained with LR+HR dataset (with LR scale 384 X 384)
+
+Backbone |  Train DB  | Config | OneDrive | GDrive
+:-|:-|:-|:-|:-
+SwinB | DUTS-TR, HRSOD-TR | [InSPyReNet_SwinB_DH_LR](../configs/extra_dataset/InSPyReNet_SwinB_DH_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EWxPZoIKALlGsfrNgUFNvxwBC8IE8jzzhPNtzcbHmTNFcg?e=e22wmy&download=1) | [Link](https://drive.google.com/file/d/1nbs6Xa7NMtcikeHFtkQRVrsHbBRHtIqC/view?usp=sharing) 
+SwinB | HRSOD-TR, UHRSD-TR | [InSPyReNet_SwinB_HU_LR](../configs/extra_dataset/InSPyReNet_SwinB_HU_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EQe-iy0AZctIkgl3o-BmVYUBn795wvii3tsnBq1fNUbc9g?e=gMZ4PV&download=1) | [Link](https://drive.google.com/file/d/1uLSIYXlRsZv4Ho0C-c87xKPhmF_b-Ll4/view?usp=sharing) 
+SwinB | DUTS-TR, HRSOD-TR, UHRSD-TR | [InSPyReNet_SwinB_DHU_LR](../configs/extra_dataset/InSPyReNet_SwinB_DHU_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfsCbnfAU1RAqCJIkj1ewRgBhFetStsGB6SMSq_UJZimjA?e=Ghuacy&download=1) | [Link](https://drive.google.com/file/d/14gRNwR7XwJ5oEcR4RWIVbYH3HEV6uBUq/view?usp=sharing) 
+
+### Trained with LR+HR dataset (with HR scale 1024 X 1024)
+
+Backbone |  Train DB  | Config | OneDrive | GDrive
+:-|:-|:-|:-|:-
+SwinB | DUTS-TR, HRSOD-TR | [InSPyReNet_SwinB_DH](../configs/extra_dataset/InSPyReNet_SwinB_DH.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EW2Qg-tMBBxNkygMj-8QgMUBiqHox5ExTOJl0LGLsn6AtA?e=Mam8Ur&download=1) | [Link](https://drive.google.com/file/d/1UBGFDUYZ9SysZr96dhsscZg7nDXt6IUD/view?usp=sharing) 
+SwinB | HRSOD-TR, UHRSD-TR | [InSPyReNet_SwinB_HU](../configs/extra_dataset/InSPyReNet_SwinB_HU.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EeE8nnCt_AdFvxxu0JsxwDgBCtGchuUka6DW9za_epX-Qw?e=U7wZu9&download=1) | [Link](https://drive.google.com/file/d/1HB02tiInEgo-pNzwqyvyV6eSN1Y2xPRJ/view?usp=sharing)
+
+
+### Trained with Dichotomous Image Segmentation dataset (DIS5K-TR) with LR (384 X 384) and HR (1024 X 1024) scale
+
+Backbone |  Train DB  | Config | OneDrive | GDrive
+:-|:-|:-|:-|:-
+SwinB | DIS5K-TR | [InSPyReNet_SwinB_DIS5K_LR](../configs/extra_dataset/InSPyReNet_SwinB_DIS5K_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ERKrQ_YeoJRHl_3HcH8ZJLoBedsa6hZlmIIf66wobZRGuw?e=EywJmS&download=1) | [Link](https://drive.google.com/file/d/1Sj7GZoocGMHyKNhFnQQc1FTs76ysJIX3/view?usp=sharing)
+SwinB | DIS5K-TR | [InSPyReNet_SwinB_DIS5K](../configs/extra_dataset/InSPyReNet_SwinB_DIS5K.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EShRbD-jZuRJiWv6DS2Us34BwgazGZvK1t4uTKvgE5379Q?e=8oVpS8) | [Link](https://drive.google.com/file/d/1aCxHMbhvj8ah77jXVgqvqImQA_Y0G-Yg/view?usp=sharing)
+
+### <img src="https://www.kindpng.com/picc/b/124-1249525_all-might-png.png" width=20px> Trained with Massive SOD Datasets (with LR (384 X 384) and HR (1024 X 1024), Not in the paper, for real-world scenarios)
+
+Backbone |  Train DB  | Config | OneDrive | GDrive
+:-|:-|:-|:-|:-
+SwinB | DUTS-TR, HRSOD-TR, UHRSD-TR, DIS-TR, DUTS-TE, DUT-OMRON, ECSSD,HKU-IS, PASCAL-S, DAVIS-S, HRSOD-TE,UHRSD-TE, FSS-1000, MSRA-10K, DIS-VD, DIS-TE1, DIS-TE2, DIS-TE3, DIS-TE4 | [Plus_Ultra_LR](../configs/extra_dataset/Plus_Ultra_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESKuh1zhToVFsIxhUUsgkbgBnu2kFXCFLRuSz1xxsKzjhA?e=02HDrm&download=1) | [Link](https://drive.google.com/file/d/1iRX-0MVbUjvAVns5MtVdng6CQlGOIo3m/view?usp=sharing)
+SwinB | DUTS-TR, HRSOD-TR, UHRSD-TR, DIS-TR, DUTS-TE, DUT-OMRON, ECSSD,HKU-IS, PASCAL-S, DAVIS-S, HRSOD-TE,UHRSD-TE, FSS-1000, MSRA-10K, DIS-VD, DIS-TE1, DIS-TE2, DIS-TE3, DIS-TE4 | [Plus_Ultra](../configs/extra_dataset/Plus_Ultra.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ET0R-yM8MfVHqI4g94AlL6AB-D6LxNajaWeDV4xbVQyh7w?e=l4JkZn) | [Link](https://drive.google.com/file/d/13oBl5MTVcWER3YU4fSxW3ATlVfueFQPY/view?usp=sharing)
+
+* We used above checkpoints for [`transparent-background`](https://github.com/plemeri/transparent-background) which is a cool background removing command-line tool / python API!
+
+## Pre-Computed Saliency Maps
+
+### Salient Object Detection benchmarks
+
+Config | DUTS-TE | DUT-OMRON | ECSSD | HKU-IS | PASCAL-S | DAVIS-S | HRSOD-TE | UHRSD-TE
+:-|:-|:-|:-|:-|:-|:-|:-|:-
+[InSPyReNet_Res2Net50](../configs/InSPyReNet_Res2Net50.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Eb0iKXGX1vxEjPhe9KGBKr0Bv7v2vv6Ua5NFybwc6aIi1w?e=oHnGyJ&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ef1HaYMvgh1EuuOL8bw3JGYB41-yo6KdTD8FGXcFZX3-Bg?e=TkW2m8&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EdEQQ8o-yI9BtTpROcuB_iIBFSIk0uBJAkNyob0WI04-kw?e=cwEj2V&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ec6LyrumVZ9PoB2Af0OW4dcBrDht0OznnwOBYiu8pdyJ4A?e=Y04Fmn&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETPijMHlTRZIjqO5H4LBknUBmy8TGDwOyUQ1H4EnIpHVOw?e=k1afrh&download=1) | N/A | N/A | N/A |
+[InSPyReNet_SwinB](../configs/InSPyReNet_SwinB.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETumLjuBantLim4kRqj4e_MBpK_X5XrTwjGQUToN8TKVjw?e=ZT8AWy&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZbwxhwT6dtHkBJrIMMjTnkBK_HaDTXgHcDSjxuswZKTZw?e=9XeE4b&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESfQK-557uZOmUwG5W49j0EBK42_7dMOaQcPsc_U1zsYlA?e=IvjkKX&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EURH96JUp55EgUHI0A8RzKoBBqvQc1nVb_a67RgwOY7f-w?e=IP9xKa&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EakMpwONph9EmnCM2rS3hn4B_TL42T6tuLjBEeEa5ndkIw?e=XksfA5&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETUCKFX0k8lAvpsDj5sT23QB2ohuE_ST7oQnWdaW7AoCIw?e=MbSmM2&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ea6kf6Kk8fpIs15WWDfJMoYBeQUeo9WXvYx9oM5yWFE1Jg?e=RNN0Ns&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EVJLvAP3HwtHksZMUolIfCABHqP7GgAWcG_1V5T_Xrnr2g?e=ct3pzo&download=1) |
+[InSPyReNet_SwinB_DH_LR](../configs/extra_dataset/InSPyReNet_SwinB_DH_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EbaXjDWFb6lGp9x5ae9mJPgBt9dkmgclq9XrXDjj4B5qSw?e=57wnhE) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EU7uNIpKPXZHrDNt3gapGfsBaUrkCj67-Paj4w_7E8xs1g?e=n0QBR1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EeBq5uU02FRLiW-0g-mQ_CsB3iHlMgAyOSY2Deu5suo9pQ?e=zjhB33) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EY5AppwMB4hFqbISFL_u5QMBeux7dQWtDeXaMMcAyLqLqQ?e=N71XVt) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfhejUtLallLroU6jjgDl-oBSHITWnkdiU6NVV95DO5YqQ?e=T6rrRW) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EesN4fRAE4JJk1aZ_tH3EBQBgirysALcgfw1Ipsa9dLe9Q?e=b5oWsg) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETpVYDijnZhInN9zBHRuwhUBSPslqe9FP0m3Eo3TWS0d5A?e=QTzfx6) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUkKVH3LSyFLq6UXuvWEVuYBEH8W8uAKgyolLRVuIUILag?e=y1SceD)
+[InSPyReNet_SwinB_HU_LR](../configs/extra_dataset/InSPyReNet_SwinB_HU_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUSblgWAwg9Plc4LCGj4TLwB-7HLEdZGJqF1jHOU55g3OA?e=2YT3zM) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESNaVX4Fh5JHn5jOfgnSWi4Bx1bc9t6pg79IoG3mrpZpAw?e=M8D0CM) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETKjTH1vcVZDu5ahRaw4cb8B7JKaPMR-0Uae1DbwarobIA?e=Qw67IZ) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EcC9-lPgAXZHs7th9DiVjygB-zPIq_1Ii6i1GpbLGc1iPQ?e=EVXKp9) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EXjyAS4XyzlPqrpaizgElioBdmgd4E81qQzj11Qm4xo5sA?e=hoOzc2) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EcSCMR033GVNotOilIzYhIsBikzb8ZzGlkuW6aSNMlUpqQ?e=TFcgvE) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUHRSEANcmVFjcS_K-PeYr0B6VPXPgb2AHFUnlJYrf3dOQ?e=unwcqV) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EU2gb0hS5kZBgKNXWqQAomsBXU-zjGCXKAzYNNk4d6EAiQ?e=pjhiN2)
+[InSPyReNet_SwinB_DHU_LR](../configs/extra_dataset/InSPyReNet_SwinB_DHU_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ee7Y647ERzxMgBoFceEEO6kBIUkIlmYHoizMj71gT37sxw?e=xDt83C) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESFGN3FCdzRAvlsW6bEaGdoBYNoJgK4DAjaS6WkVVyI_QQ?e=nYHklV) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EeONq5kOirRCkErrb6fFqd8B4w4SMZXBY1Q2mJvZcRsGdQ?e=K7fwQt) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ea5P4CBBQatPiUzsH53lckoB0k23haePzuERBfyJXaCbBg?e=AZ96mc) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ea2pezB7eo1BraeBpIA8YZoBkVY38rRa3KrwSIzY1cn2dQ?e=o121S6) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EfNDzH8O54pGtnAxit_hUjUBK9poVq4sxxnJjSG7PUQCkw?e=OWt7k8) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESfZaII0pO9IqqL1FkpjIuAB8SGxLcslJeWTuKQxPNFIVA?e=Ce1CWg) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EYsQqf1GKShJhBnkZ6gD5PABPOcjRcUGSfvTbe-wYh2O2Q?e=t4Xlxv)
+[InSPyReNet_SwinB_DH](../configs/extra_dataset/InSPyReNet_SwinB_DH.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EWNaRqtzhtNFhMVfLcoyfqQBw35M8q8bxME3yZyhkTtc7Q?e=jrJe3v) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZF5_s8JfR9HqGBUZHSM_j4BVVMONp38_gJ1ekEdvlM-qQ?e=0chMdl) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZuaFObyNOtKg0W5cM7bqPYBZYGg7Z3V3i4sClI6bU_ntA?e=BxxQI7) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EY-yEnNNNT5KpiURhiDAkDEBMkiA1QwQ_T0wB1UC75GXVg?e=Lle02B) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EYfbsgWhm7lAlX_wj_WZZowBV_-l-UvvThC4LJEKpV0BQQ?e=zTiKpI) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Ef0GUP7c0bBBonHlqgB988YB0rgxCFq3oo0u8xCN8wfyyQ?e=LCb8UV) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EVUZBnRpa35AmrvdUybsQDMBzMZvuJWe5tT7635lh9MHDQ?e=FlpQW1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETfZ_zdrDvhOh21u2mqVhigBSxn3vlfKVIwXhRfzzSSFzA?e=kXBBi9)
+[InSPyReNet_SwinB_HU](../configs/extra_dataset/InSPyReNet_SwinB_HU.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZq4JUACKCBMk2bn4yoWz6sBOKrSFTPfL7d5xopc1uDw_A?e=RtVHSl) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ETJaqoSPaYtNkc8eSGDeKzMBbjbuOAWgJwG4q52bW87aew?e=Pguh4b) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZAeCI6BPMdNsicnQ-m1pVEBwAhOiIcbelhOMoRGXGEvVA?e=BQKd7Q) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EVmGvZGz54JOvrIymLsSwq4Bpos3vWSXZm3oV7-qmGZgHA?e=4UhDgv) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ERHDUybOh4ZKkqWZpcu7MiMBFuTK6wACkKUZaNeEQGbCNQ?e=GCQnoe) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESPmZXTnfO5CrCoo_0OADxgBt_3FoU5mSFoSE4QWbWxumQ?e=HAsAYz) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EdTnUwEeMZNBrPSBBbGZKQcBmVshSTfca9qz_BqNpAUpOg?e=HsJ4Gx) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ET48owfVQEdImrh0V4gx8_ABsYXgbIJqtpq77aK_U28VwQ?e=h8er3H)
+
+### DIS5K Results
+
+Config | DIS-VD | DIS-TE1 | DIS-TE2 | DIS-TE3 | DIS-TE4
+:-|:-|:-|:-|:-|:-|
+[InSPyReNet_SwinB_DIS5K_LR](../configs/extra_dataset/InSPyReNet_SwinB_DIS5K_LR.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUbzddb_QRRCtnXC8Xl6vZoBC6IqOfom52BWbzOYk-b2Ow?e=aqJYi1&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ESeW_SOD26tHjBLymmgFaXwBIJlljzNycaGWXLpOp_d_kA?e=2EyMai&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EYWT5fZDjI5Bn-lr-iQM1TsB1num0-UqfJC1TIv-LuOXoA?e=jCcnty&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EQXm1DEBfaNJmH0B-A3o23kBn4v5j53kP2nF9CpG9SQkyw?e=lEUiZh&download=1) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EZeH2ufGsFZIoUh6D8Rtv88BBF_ddQXav4xYXXRP_ayEAg?e=AMzIp8&download=1)
+[InSPyReNet_SwinB_DIS5K](../configs/extra_dataset/InSPyReNet_SwinB_DIS5K.yaml) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EUisiphB_W5BjgfpIYu9oNgB_fY4XxL-MhR2gR-ZZUt49Q?e=gqorYs) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EW2y54ROYIZFlEq5ilRFwOsBSrIm2-HGsUHPHykaJvUBfA?e=797fxr) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/ER6yEGZkgWVOsL-mauYgPyoBDIoU0Mck-twEBgQi5g3Mxw?e=0yJZTT) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/Edvzj0iZ8hdDthm4Q-p2YHgBhP1X5z4AAccAoUasr2nihA?e=1dognG) | [Link](https://postechackr-my.sharepoint.com/:u:/g/personal/taehoon1018_postech_ac_kr/EWwMORYg8DlCgGGqOQFThZ8BgIU9wV9-0DwLrKldQl7N8w?e=nCwuqy)

lib/InSPyReNet.py

@@ -0,0 +1,199 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+from lib.optim import *
+from lib.modules.layers import *
+from lib.modules.context_module import *
+from lib.modules.attention_module import *
+from lib.modules.decoder_module import *
+
+from lib.backbones.Res2Net_v1b import res2net50_v1b_26w_4s
+from lib.backbones.SwinTransformer import SwinB
+
+class InSPyReNet(nn.Module):
+    def __init__(self, backbone, in_channels, depth=64, base_size=[384, 384], threshold=512, **kwargs):
+        super(InSPyReNet, self).__init__()
+        self.backbone = backbone
+        self.in_channels = in_channels
+        self.depth = depth
+        self.base_size = base_size
+        self.threshold = threshold
+        
+        self.context1 = PAA_e(self.in_channels[0], self.depth, base_size=self.base_size, stage=0)
+        self.context2 = PAA_e(self.in_channels[1], self.depth, base_size=self.base_size, stage=1)
+        self.context3 = PAA_e(self.in_channels[2], self.depth, base_size=self.base_size, stage=2)
+        self.context4 = PAA_e(self.in_channels[3], self.depth, base_size=self.base_size, stage=3)
+        self.context5 = PAA_e(self.in_channels[4], self.depth, base_size=self.base_size, stage=4)
+
+        self.decoder = PAA_d(self.depth * 3, depth=self.depth, base_size=base_size, stage=2)
+
+        self.attention0 = SICA(self.depth    , depth=self.depth, base_size=self.base_size, stage=0, lmap_in=True)
+        self.attention1 = SICA(self.depth * 2, depth=self.depth, base_size=self.base_size, stage=1, lmap_in=True)
+        self.attention2 = SICA(self.depth * 2, depth=self.depth, base_size=self.base_size, stage=2              )
+
+        self.sod_loss_fn = lambda x, y: weighted_bce_loss_with_logits(x, y, reduction='mean') + iou_loss_with_logits(x, y, reduction='mean')
+        self.pc_loss_fn  = nn.L1Loss()
+
+        self.ret = lambda x, target: F.interpolate(x, size=target.shape[-2:], mode='bilinear', align_corners=False)
+        self.res = lambda x, size: F.interpolate(x, size=size, mode='bilinear', align_corners=False)
+        self.des = lambda x, size: F.interpolate(x, size=size, mode='nearest')
+        
+        self.image_pyramid = ImagePyramid(7, 1)
+        
+        self.transition0 = Transition(17)
+        self.transition1 = Transition(9)
+        self.transition2 = Transition(5)
+        
+        self.forward = self.forward_inference
+        
+    def to(self, device):
+        self.image_pyramid.to(device)
+        self.transition0.to(device)
+        self.transition1.to(device)
+        self.transition2.to(device)
+        super(InSPyReNet, self).to(device)
+        return self
+    
+    def cuda(self, idx=None):
+        if idx is None:
+            idx = torch.cuda.current_device()
+            
+        self.to(device="cuda:{}".format(idx))
+        return self
+    
+    def train(self, mode=True):
+        super(InSPyReNet, self).train(mode)
+        self.forward = self.forward_train
+        return self
+    
+    def eval(self):
+        super(InSPyReNet, self).train(False)
+        self.forward = self.forward_inference
+        return self
+    
+    def forward_inspyre(self, x):
+        B, _, H, W = x.shape
+    
+        x1, x2, x3, x4, x5 = self.backbone(x)
+        
+        x1 = self.context1(x1) #4
+        x2 = self.context2(x2) #4
+        x3 = self.context3(x3) #8
+        x4 = self.context4(x4) #16
+        x5 = self.context5(x5) #32
+
+        f3, d3 = self.decoder([x3, x4, x5]) #16
+
+        f3 = self.res(f3, (H // 4,  W // 4 ))
+        f2, p2 = self.attention2(torch.cat([x2, f3], dim=1), d3.detach())
+        d2 = self.image_pyramid.reconstruct(d3.detach(), p2) #4
+
+        x1 = self.res(x1, (H // 2, W // 2))
+        f2 = self.res(f2, (H // 2, W // 2))
+        f1, p1 = self.attention1(torch.cat([x1, f2], dim=1), d2.detach(), p2.detach()) #2
+        d1 = self.image_pyramid.reconstruct(d2.detach(), p1) #2
+        
+        f1 = self.res(f1, (H, W))
+        _, p0 = self.attention0(f1, d1.detach(), p1.detach()) #2
+        d0 = self.image_pyramid.reconstruct(d1.detach(), p0) #2
+        
+        out = dict()
+        out['saliency'] = [d3, d2, d1, d0]
+        out['laplacian'] = [p2, p1, p0]
+        
+        return out
+    
+    def forward_train(self, sample):
+        x = sample['image']
+        B, _, H, W = x.shape
+        out = self.forward_inspyre(x)
+        
+        d3, d2, d1, d0 = out['saliency']
+        p2, p1, p0     = out['laplacian']
+        
+        if type(sample) == dict and 'gt' in sample.keys() and sample['gt'] is not None:
+            y = sample['gt']
+            
+            y1 = self.image_pyramid.reduce(y)
+            y2 = self.image_pyramid.reduce(y1)
+            y3 = self.image_pyramid.reduce(y2)
+
+            loss =  self.pc_loss_fn(self.des(d3, (H, W)), self.des(self.image_pyramid.reduce(d2), (H, W)).detach()) * 0.0001
+            loss += self.pc_loss_fn(self.des(d2, (H, W)), self.des(self.image_pyramid.reduce(d1), (H, W)).detach()) * 0.0001
+            loss += self.pc_loss_fn(self.des(d1, (H, W)), self.des(self.image_pyramid.reduce(d0), (H, W)).detach()) * 0.0001
+            
+            loss +=  self.sod_loss_fn(self.des(d3, (H, W)), self.des(y3, (H, W)))
+            loss += self.sod_loss_fn(self.des(d2, (H, W)), self.des(y2, (H, W)))
+            loss += self.sod_loss_fn(self.des(d1, (H, W)), self.des(y1, (H, W)))
+            loss += self.sod_loss_fn(self.des(d0, (H, W)), self.des(y, (H, W)))
+            
+        else:
+            loss = 0
+            
+        pred = torch.sigmoid(d0)
+        pred = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8)
+
+        sample['pred'] = pred
+        sample['loss'] = loss
+        sample['saliency'] = [d3, d2, d1, d0]
+        sample['laplacian'] = [p2, p1, p0]
+        return sample
+    
+    def forward_inference(self, sample):
+        B, _, H, W = sample['image'].shape
+        
+        if self.threshold is None:
+            out = self.forward_inspyre(sample['image'])
+            d3, d2, d1, d0 = out['saliency']
+            p2, p1, p0     = out['laplacian']
+            
+        elif (H <= self.threshold or W <= self.threshold):
+            if 'image_resized' in sample.keys():
+                out = self.forward_inspyre(sample['image_resized'])
+            else:
+                out = self.forward_inspyre(sample['image'])
+            d3, d2, d1, d0 = out['saliency']
+            p2, p1, p0     = out['laplacian']
+        
+        else:
+            # LR Saliency Pyramid
+            lr_out = self.forward_inspyre(sample['image_resized'])
+            lr_d3, lr_d2, lr_d1, lr_d0 = lr_out['saliency']
+            lr_p2, lr_p1, lr_p0      = lr_out['laplacian']
+                
+            # HR Saliency Pyramid
+            hr_out = self.forward_inspyre(sample['image'])
+            hr_d3, hr_d2, hr_d1, hr_d0 = hr_out['saliency']
+            hr_p2, hr_p1, hr_p0      = hr_out['laplacian']
+            
+            # Pyramid Blending
+            d3 = self.ret(lr_d0, hr_d3) 
+            
+            t2 = self.ret(self.transition2(d3), hr_p2)
+            p2 = t2 * hr_p2
+            d2 = self.image_pyramid.reconstruct(d3, p2)
+            
+            t1 = self.ret(self.transition1(d2), hr_p1)
+            p1 = t1 * hr_p1
+            d1 = self.image_pyramid.reconstruct(d2, p1)
+            
+            t0 = self.ret(self.transition0(d1), hr_p0)
+            p0 = t0 * hr_p0
+            d0 = self.image_pyramid.reconstruct(d1, p0)
+            
+        pred = torch.sigmoid(d0)
+        pred = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8)
+
+        sample['pred'] = pred
+        sample['loss'] = 0
+        sample['saliency'] = [d3, d2, d1, d0]
+        sample['laplacian'] = [p2, p1, p0]
+        return sample
+    
+def InSPyReNet_Res2Net50(depth, pretrained, base_size, **kwargs):
+    return InSPyReNet(res2net50_v1b_26w_4s(pretrained=pretrained), [64, 256, 512, 1024, 2048], depth, base_size, **kwargs)
+
+def InSPyReNet_SwinB(depth, pretrained, base_size, **kwargs):
+    return InSPyReNet(SwinB(pretrained=pretrained), [128, 128, 256, 512, 1024], depth, base_size, **kwargs)

lib/__init__.py

@@ -0,0 +1 @@
+from lib.InSPyReNet import InSPyReNet_Res2Net50, InSPyReNet_SwinB

lib/backbones/Res2Net_v1b.py

@@ -0,0 +1,268 @@
+import torch.nn as nn
+import math
+import torch.utils.model_zoo as model_zoo
+import torch
+import torch.nn.functional as F
+
+__all__ = ['Res2Net', 'res2net50_v1b',
+           'res2net101_v1b', 'res2net50_v1b_26w_4s']
+
+model_urls = {
+    'res2net50_v1b_26w_4s': 'https://shanghuagao.oss-cn-beijing.aliyuncs.com/res2net/res2net50_v1b_26w_4s-3cf99910.pth',
+    'res2net101_v1b_26w_4s': 'https://shanghuagao.oss-cn-beijing.aliyuncs.com/res2net/res2net101_v1b_26w_4s-0812c246.pth',
+}
+class Bottle2neck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, baseWidth=26, scale=4, stype='normal'):
+        super(Bottle2neck, self).__init__()
+
+        width = int(math.floor(planes * (baseWidth / 64.0)))
+        self.conv1 = nn.Conv2d(inplanes, width * scale,
+                               kernel_size=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width * scale)
+
+        if scale == 1:
+            self.nums = 1
+        else:
+            self.nums = scale - 1
+        if stype == 'stage':
+            self.pool = nn.AvgPool2d(kernel_size=3, stride=stride, padding=1)
+        convs = []
+        bns = []
+        for i in range(self.nums):
+            convs.append(nn.Conv2d(width, width, kernel_size=3, stride=stride,
+                                   dilation=dilation, padding=dilation, bias=False))
+            bns.append(nn.BatchNorm2d(width))
+        self.convs = nn.ModuleList(convs)
+        self.bns = nn.ModuleList(bns)
+
+        self.conv3 = nn.Conv2d(width * scale, planes *
+                               self.expansion, kernel_size=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stype = stype
+        self.scale = scale
+        self.width = width
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        spx = torch.split(out, self.width, 1)
+        for i in range(self.nums):
+            if i == 0 or self.stype == 'stage':
+                sp = spx[i]
+            else:
+                sp = sp + spx[i]
+            sp = self.convs[i](sp)
+            sp = self.relu(self.bns[i](sp))
+            if i == 0:
+                out = sp
+            else:
+                out = torch.cat((out, sp), 1)
+        if self.scale != 1 and self.stype == 'normal':
+            out = torch.cat((out, spx[self.nums]), 1)
+        elif self.scale != 1 and self.stype == 'stage':
+            out = torch.cat((out, self.pool(spx[self.nums])), 1)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+
+class Res2Net(nn.Module):
+
+    def __init__(self, block, layers, baseWidth=26, scale=4, num_classes=1000, output_stride=32):
+        self.inplanes = 64
+        super(Res2Net, self).__init__()
+        self.baseWidth = baseWidth
+        self.scale = scale
+        self.output_stride = output_stride
+        if self.output_stride == 8:
+            self.grid = [1, 2, 1]
+            self.stride = [1, 2, 1, 1]
+            self.dilation = [1, 1, 2, 4]
+        elif self.output_stride == 16:
+            self.grid = [1, 2, 4]
+            self.stride = [1, 2, 2, 1]
+            self.dilation = [1, 1, 1, 2]
+        elif self.output_stride == 32:
+            self.grid = [1, 2, 4]
+            self.stride = [1, 2, 2, 2]
+            self.dilation = [1, 1, 2, 4]
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(3, 32, 3, 2, 1, bias=False),
+            nn.BatchNorm2d(32),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(32, 32, 3, 1, 1, bias=False),
+            nn.BatchNorm2d(32),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(32, 64, 3, 1, 1, bias=False)
+        )
+        self.bn1 = nn.BatchNorm2d(64)
+        self.relu = nn.ReLU()
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+        self.layer1 = self._make_layer(
+            block, 64,  layers[0], stride=self.stride[0], dilation=self.dilation[0])
+        self.layer2 = self._make_layer(
+            block, 128, layers[1], stride=self.stride[1], dilation=self.dilation[1])
+        self.layer3 = self._make_layer(
+            block, 256, layers[2], stride=self.stride[2], dilation=self.dilation[2])
+        self.layer4 = self._make_layer(
+            block, 512, layers[3], stride=self.stride[3], dilation=self.dilation[3], grid=self.grid)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(
+                    m.weight, mode='fan_out', nonlinearity='relu')
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+    def _make_layer(self, block, planes, blocks, stride=1, dilation=1, grid=None):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                nn.AvgPool2d(kernel_size=stride, stride=stride,
+                             ceil_mode=True, count_include_pad=False),
+                nn.Conv2d(self.inplanes, planes * block.expansion,
+                          kernel_size=1, stride=1, bias=False),
+                nn.BatchNorm2d(planes * block.expansion),
+            )
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, dilation, downsample=downsample,
+                            stype='stage', baseWidth=self.baseWidth, scale=self.scale))
+        self.inplanes = planes * block.expansion
+
+        if grid is not None:
+            assert len(grid) == blocks
+        else:
+            grid = [1] * blocks
+
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes, dilation=dilation *
+                                grid[i], baseWidth=self.baseWidth, scale=self.scale))
+
+        return nn.Sequential(*layers)
+
+    def change_stride(self, output_stride=16):
+        if output_stride == self.output_stride:
+            return
+        else:
+            self.output_stride = output_stride
+            if self.output_stride == 8:
+                self.grid = [1, 2, 1]
+                self.stride = [1, 2, 1, 1]
+                self.dilation = [1, 1, 2, 4]
+            elif self.output_stride == 16:
+                self.grid = [1, 2, 4]
+                self.stride = [1, 2, 2, 1]
+                self.dilation = [1, 1, 1, 2]
+            elif self.output_stride == 32:
+                self.grid = [1, 2, 4]
+                self.stride = [1, 2, 2, 2]
+                self.dilation = [1, 1, 2, 4]
+
+            for i, layer in enumerate([self.layer1, self.layer2, self.layer3, self.layer4]):
+                for j, block in enumerate(layer):
+                    if block.downsample is not None:
+                        block.downsample[0].kernel_size = (
+                            self.stride[i], self.stride[i])
+                        block.downsample[0].stride = (
+                            self.stride[i], self.stride[i])
+                        if hasattr(block, 'pool'):
+                            block.pool.stride = (
+                                self.stride[i], self.stride[i])
+                        for conv in block.convs:
+                            conv.stride = (self.stride[i], self.stride[i])
+                    for conv in block.convs:
+                        d = self.dilation[i] if i != 3 else self.dilation[i] * \
+                            self.grid[j]
+                        conv.dilation = (d, d)
+                        conv.padding = (d, d)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+        
+        out = [x]
+
+        x = self.layer1(x)
+        out.append(x)
+        x = self.layer2(x)
+        out.append(x)
+        x = self.layer3(x)
+        out.append(x)
+        x = self.layer4(x)
+        out.append(x)
+
+        return out
+
+
+def res2net50_v1b(pretrained=False, **kwargs):
+    model = Res2Net(Bottle2neck, [3, 4, 6, 3], baseWidth=26, scale=4, **kwargs)
+    if pretrained:
+        model.load_state_dict(model_zoo.load_url(
+            model_urls['res2net50_v1b_26w_4s']))
+    
+    return model
+
+
+def res2net101_v1b(pretrained=False, **kwargs):
+    model = Res2Net(Bottle2neck, [3, 4, 23, 3],
+                    baseWidth=26, scale=4, **kwargs)
+    if pretrained:
+        model.load_state_dict(model_zoo.load_url(
+            model_urls['res2net101_v1b_26w_4s']))
+    
+    return model
+
+
+def res2net50_v1b_26w_4s(pretrained=True, **kwargs):
+    model = Res2Net(Bottle2neck, [3, 4, 6, 3], baseWidth=26, scale=4, **kwargs)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/res2net50_v1b_26w_4s-3cf99910.pth', map_location='cpu'))
+    
+    return model
+
+
+def res2net101_v1b_26w_4s(pretrained=True, **kwargs):
+    model = Res2Net(Bottle2neck, [3, 4, 23, 3],
+                    baseWidth=26, scale=4, **kwargs)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/res2net101_v1b_26w_4s-0812c246.pth', map_location='cpu'))
+    
+    return model
+
+
+def res2net152_v1b_26w_4s(pretrained=False, **kwargs):
+    model = Res2Net(Bottle2neck, [3, 8, 36, 3],
+                    baseWidth=26, scale=4, **kwargs)
+    if pretrained:
+        model.load_state_dict(model_zoo.load_url(
+            model_urls['res2net152_v1b_26w_4s']))
+    
+    return model
+
+
+if __name__ == '__main__':
+    images = torch.rand(1, 3, 224, 224).cuda(0)
+    model = res2net50_v1b_26w_4s(pretrained=True)
+    model = model.cuda(0)
+    print(model(images).size())

lib/backbones/SwinTransformer.py

@@ -0,0 +1,652 @@
+# --------------------------------------------------------
+# Swin Transformer
+# Copyright (c) 2021 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu, Yutong Lin, Yixuan Wei
+# --------------------------------------------------------
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+import numpy as np
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+
+class Mlp(nn.Module):
+    """ Multilayer perceptron."""
+
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+
+    Returns:
+        x: (B, H, W, C)
+    """
+    B = int(windows.shape[0] / (H * W / window_size / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+
+class WindowAttention(nn.Module):
+    """ Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        # define a parameter table of relative position bias
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads))  # 2*Wh-1 * 2*Ww-1, nH
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        trunc_normal_(self.relative_position_bias_table, std=.02)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """ Forward function.
+
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))
+
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class SwinTransformerBlock(nn.Module):
+    """ Swin Transformer Block.
+
+    Args:
+        dim (int): Number of input channels.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, dim, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,
+                 act_layer=nn.GELU, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size
+        self.mlp_ratio = mlp_ratio
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        self.H = None
+        self.W = None
+
+    def forward(self, x, mask_matrix):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+            mask_matrix: Attention mask for cyclic shift.
+        """
+        B, L, C = x.shape
+        H, W = self.H, self.W
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = self.norm1(x)
+        x = x.view(B, H, W, C)
+
+        # pad feature maps to multiples of window size
+        pad_l = pad_t = 0
+        pad_r = (self.window_size - W % self.window_size) % self.window_size
+        pad_b = (self.window_size - H % self.window_size) % self.window_size
+        x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
+        _, Hp, Wp, _ = x.shape
+
+        # cyclic shift
+        if self.shift_size > 0:
+            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+            attn_mask = mask_matrix
+        else:
+            shifted_x = x
+            attn_mask = None
+
+        # partition windows
+        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        attn_windows = self.attn(x_windows, mask=attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+        shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp)  # B H' W' C
+
+        # reverse cyclic shift
+        if self.shift_size > 0:
+            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+        else:
+            x = shifted_x
+
+        if pad_r > 0 or pad_b > 0:
+            x = x[:, :H, :W, :].contiguous()
+
+        x = x.view(B, H * W, C)
+
+        # FFN
+        x = shortcut + self.drop_path(x)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+
+        return x
+
+
+class PatchMerging(nn.Module):
+    """ Patch Merging Layer
+
+    Args:
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+    def __init__(self, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(4 * dim)
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        x = x.view(B, H, W, C)
+
+        # padding
+        pad_input = (H % 2 == 1) or (W % 2 == 1)
+        if pad_input:
+            x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))
+
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.norm(x)
+        x = self.reduction(x)
+
+        return x
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+
+    Args:
+        dim (int): Number of feature channels
+        depth (int): Depths of this stage.
+        num_heads (int): Number of attention head.
+        window_size (int): Local window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 dim,
+                 depth,
+                 num_heads,
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 norm_layer=nn.LayerNorm,
+                 downsample=None,
+                 use_checkpoint=False):
+        super().__init__()
+        self.window_size = window_size
+        self.shift_size = window_size // 2
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(
+                dim=dim,
+                num_heads=num_heads,
+                window_size=window_size,
+                shift_size=0 if (i % 2 == 0) else window_size // 2,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop,
+                attn_drop=attn_drop,
+                drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                norm_layer=norm_layer)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+
+        # calculate attention mask for SW-MSA
+        Hp = int(np.ceil(H / self.window_size)) * self.window_size
+        Wp = int(np.ceil(W / self.window_size)) * self.window_size
+        img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device)  # 1 Hp Wp 1
+        h_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        w_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        cnt = 0
+        for h in h_slices:
+            for w in w_slices:
+                img_mask[:, h, w, :] = cnt
+                cnt += 1
+
+        mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+        mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+        attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+        attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+
+        for blk in self.blocks:
+            blk.H, blk.W = H, W
+            if self.use_checkpoint:
+                x = checkpoint.checkpoint(blk, x, attn_mask)
+            else:
+                x = blk(x, attn_mask)
+        if self.downsample is not None:
+            x_down = self.downsample(x, H, W)
+            Wh, Ww = (H + 1) // 2, (W + 1) // 2
+            return x, H, W, x_down, Wh, Ww
+        else:
+            return x, H, W, x, H, W
+
+
+class PatchEmbed(nn.Module):
+    """ Image to Patch Embedding
+
+    Args:
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        patch_size = to_2tuple(patch_size)
+        self.patch_size = patch_size
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        """Forward function."""
+        # padding
+        _, _, H, W = x.size()
+        if W % self.patch_size[1] != 0:
+            x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))
+        if H % self.patch_size[0] != 0:
+            x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))
+
+        x = self.proj(x)  # B C Wh Ww
+        if self.norm is not None:
+            Wh, Ww = x.size(2), x.size(3)
+            x = x.flatten(2).transpose(1, 2)
+            x = self.norm(x)
+            x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)
+
+        return x
+
+
+class SwinTransformer(nn.Module):
+    """ Swin Transformer backbone.
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+
+    Args:
+        pretrain_img_size (int): Input image size for training the pretrained model,
+            used in absolute postion embedding. Default 224.
+        patch_size (int | tuple(int)): Patch size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        depths (tuple[int]): Depths of each Swin Transformer stage.
+        num_heads (tuple[int]): Number of attention head of each stage.
+        window_size (int): Window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set.
+        drop_rate (float): Dropout rate.
+        attn_drop_rate (float): Attention dropout rate. Default: 0.
+        drop_path_rate (float): Stochastic depth rate. Default: 0.2.
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False.
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True.
+        out_indices (Sequence[int]): Output from which stages.
+        frozen_stages (int): Stages to be frozen (stop grad and set eval mode).
+            -1 means not freezing any parameters.
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 pretrain_img_size=224,
+                 patch_size=4,
+                 in_chans=3,
+                 embed_dim=96,
+                 depths=[2, 2, 6, 2],
+                 num_heads=[3, 6, 12, 24],
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.2,
+                 norm_layer=nn.LayerNorm,
+                 ape=False,
+                 patch_norm=True,
+                 out_indices=(0, 1, 2, 3),
+                 frozen_stages=-1,
+                 use_checkpoint=False):
+        super().__init__()
+
+        self.pretrain_img_size = pretrain_img_size
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dim
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,
+            norm_layer=norm_layer if self.patch_norm else None)
+
+        # absolute position embedding
+        if self.ape:
+            pretrain_img_size = to_2tuple(pretrain_img_size)
+            patch_size = to_2tuple(patch_size)
+            patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]]
+
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]))
+            trunc_normal_(self.absolute_pos_embed, std=.02)
+
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        # stochastic depth
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(
+                dim=int(embed_dim * 2 ** i_layer),
+                depth=depths[i_layer],
+                num_heads=num_heads[i_layer],
+                window_size=window_size,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                norm_layer=norm_layer,
+                downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                use_checkpoint=use_checkpoint)
+            self.layers.append(layer)
+
+        num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]
+        self.num_features = num_features
+
+        # add a norm layer for each output
+        for i_layer in out_indices:
+            layer = norm_layer(num_features[i_layer])
+            layer_name = f'norm{i_layer}'
+            self.add_module(layer_name, layer)
+
+        self._freeze_stages()
+
+    def _freeze_stages(self):
+        if self.frozen_stages >= 0:
+            self.patch_embed.eval()
+            for param in self.patch_embed.parameters():
+                param.requires_grad = False
+
+        if self.frozen_stages >= 1 and self.ape:
+            self.absolute_pos_embed.requires_grad = False
+
+        if self.frozen_stages >= 2:
+            self.pos_drop.eval()
+            for i in range(0, self.frozen_stages - 1):
+                m = self.layers[i]
+                m.eval()
+                for param in m.parameters():
+                    param.requires_grad = False
+
+    def init_weights(self, pretrained=None):
+        """Initialize the weights in backbone.
+
+        Args:
+            pretrained (str, optional): Path to pre-trained weights.
+                Defaults to None.
+        """
+
+        def _init_weights(m):
+            if isinstance(m, nn.Linear):
+                trunc_normal_(m.weight, std=.02)
+                if isinstance(m, nn.Linear) and m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.LayerNorm):
+                nn.init.constant_(m.bias, 0)
+                nn.init.constant_(m.weight, 1.0)
+
+        if isinstance(pretrained, str):
+            self.apply(_init_weights)
+            logger = get_root_logger()
+            load_checkpoint(self, pretrained, strict=False, logger=logger)
+        elif pretrained is None:
+            self.apply(_init_weights)
+        else:
+            raise TypeError('pretrained must be a str or None')
+
+    def forward(self, x):
+        """Forward function."""
+        x = self.patch_embed(x)
+
+        Wh, Ww = x.size(2), x.size(3)
+        if self.ape:
+            # interpolate the position embedding to the corresponding size
+            absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic')
+            x = (x + absolute_pos_embed) # B Wh*Ww C
+            
+        outs = [x.contiguous()]
+        x = x.flatten(2).transpose(1, 2)
+        x = self.pos_drop(x)
+        for i in range(self.num_layers):
+            layer = self.layers[i]
+            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
+
+            if i in self.out_indices:
+                norm_layer = getattr(self, f'norm{i}')
+                x_out = norm_layer(x_out)
+
+                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
+                outs.append(out)
+
+        return tuple(outs)
+
+    def train(self, mode=True):
+        """Convert the model into training mode while keep layers freezed."""
+        super(SwinTransformer, self).train(mode)
+        self._freeze_stages()
+
+def SwinT(pretrained=True):
+    model = SwinTransformer(embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/swin_tiny_patch4_window7_224.pth', map_location='cpu')['model'], strict=False)
+        
+    return model
+
+def SwinS(pretrained=True):
+    model = SwinTransformer(embed_dim=96, depths=[2, 2, 18, 2], num_heads=[3, 6, 12, 24], window_size=7)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/swin_small_patch4_window7_224.pth', map_location='cpu')['model'], strict=False)
+        
+    return model
+
+def SwinB(pretrained=True):
+    model = SwinTransformer(embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/swin_base_patch4_window12_384_22kto1k.pth', map_location='cpu')['model'], strict=False)
+        
+    return model
+
+def SwinL(pretrained=True):
+    model = SwinTransformer(embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12)
+    if pretrained is True:
+        model.load_state_dict(torch.load('data/backbone_ckpt/swin_large_patch4_window12_384_22kto1k.pth', map_location='cpu')['model'], strict=False)
+
+    return model
+

lib/modules/attention_module.py

@@ -0,0 +1,103 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.nn.parameter import Parameter
+from operator import xor
+from typing import Optional
+
+from lib.modules.layers import *
+from utils.misc import *
+class SICA(nn.Module):
+    def __init__(self, in_channel, out_channel=1, depth=64, base_size=None, stage=None, lmap_in=False):
+        super(SICA, self).__init__()
+        self.in_channel = in_channel
+        self.depth = depth
+        self.lmap_in = lmap_in
+        if base_size is not None and stage is not None:
+            self.stage_size = (base_size[0] // (2 ** stage), base_size[1] // (2 ** stage))
+        else:
+            self.stage_size = None
+        
+        self.conv_query = nn.Sequential(Conv2d(in_channel, depth, 3, relu=True),
+                                        Conv2d(depth, depth, 3, relu=True))
+        self.conv_key   = nn.Sequential(Conv2d(in_channel, depth, 1, relu=True),
+                                        Conv2d(depth, depth, 1, relu=True))
+        self.conv_value = nn.Sequential(Conv2d(in_channel, depth, 1, relu=True),
+                                        Conv2d(depth, depth, 1, relu=True))
+
+        if self.lmap_in is True:
+            self.ctx = 5
+        else:
+            self.ctx = 3
+
+        self.conv_out1 = Conv2d(depth, depth, 3, relu=True)
+        self.conv_out2 = Conv2d(in_channel + depth, depth, 3, relu=True)
+        self.conv_out3 = Conv2d(depth, depth, 3, relu=True)
+        self.conv_out4 = Conv2d(depth, out_channel, 1)
+
+        self.threshold = Parameter(torch.tensor([0.5]))
+        
+        if self.lmap_in is True:
+            self.lthreshold = Parameter(torch.tensor([0.5]))
+
+    def forward(self, x, smap, lmap: Optional[torch.Tensor]=None):
+        assert not xor(self.lmap_in is True, lmap is not None)
+        b, c, h, w = x.shape
+        
+        # compute class probability
+        smap = F.interpolate(smap, size=x.shape[-2:], mode='bilinear', align_corners=False)
+        smap = torch.sigmoid(smap)
+        p = smap - self.threshold
+
+        fg = torch.clip(p, 0, 1) # foreground
+        bg = torch.clip(-p, 0, 1) # background
+        cg = self.threshold - torch.abs(p) # confusion area
+
+        if self.lmap_in is True and lmap is not None:
+            lmap = F.interpolate(lmap, size=x.shape[-2:], mode='bilinear', align_corners=False)
+            lmap = torch.sigmoid(lmap)
+            lp = lmap - self.lthreshold
+            fp = torch.clip(lp, 0, 1) # foreground
+            bp = torch.clip(-lp, 0, 1) # background
+
+            prob = [fg, bg, cg, fp, bp]
+        else:
+            prob = [fg, bg, cg]
+
+        prob = torch.cat(prob, dim=1)
+
+        # reshape feature & prob
+        if self.stage_size is not None:
+            shape = self.stage_size
+            shape_mul = self.stage_size[0] * self.stage_size[1]
+        else:
+            shape = (h, w)
+            shape_mul = h * w        
+        
+        f = F.interpolate(x, size=shape, mode='bilinear', align_corners=False).view(b, shape_mul, -1)
+        prob = F.interpolate(prob, size=shape, mode='bilinear', align_corners=False).view(b, self.ctx, shape_mul)
+        
+        # compute context vector
+        context = torch.bmm(prob, f).permute(0, 2, 1).unsqueeze(3) # b, 3, c
+
+        # k q v compute
+        query = self.conv_query(x).view(b, self.depth, -1).permute(0, 2, 1)
+        key = self.conv_key(context).view(b, self.depth, -1)
+        value = self.conv_value(context).view(b, self.depth, -1).permute(0, 2, 1)
+
+        # compute similarity map
+        sim = torch.bmm(query, key) # b, hw, c x b, c, 2
+        sim = (self.depth ** -.5) * sim
+        sim = F.softmax(sim, dim=-1)
+
+        # compute refined feature
+        context = torch.bmm(sim, value).permute(0, 2, 1).contiguous().view(b, -1, h, w)
+        context = self.conv_out1(context)
+        
+        x = torch.cat([x, context], dim=1)
+        x = self.conv_out2(x)
+        x = self.conv_out3(x)
+        out = self.conv_out4(x)
+
+        return x, out

lib/modules/context_module.py

@@ -0,0 +1,54 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .layers import *
+
+class PAA_kernel(nn.Module):
+    def __init__(self, in_channel, out_channel, receptive_size, stage_size=None):
+        super(PAA_kernel, self).__init__()
+        self.conv0 = Conv2d(in_channel, out_channel, 1)
+        self.conv1 = Conv2d(out_channel, out_channel, kernel_size=(1, receptive_size))
+        self.conv2 = Conv2d(out_channel, out_channel, kernel_size=(receptive_size, 1))
+        self.conv3 = Conv2d(out_channel, out_channel, 3, dilation=receptive_size)
+        self.Hattn = SelfAttention(out_channel, 'h', stage_size[0] if stage_size is not None else None)
+        self.Wattn = SelfAttention(out_channel, 'w', stage_size[1] if stage_size is not None else None)
+
+    def forward(self, x):
+        x = self.conv0(x)
+        x = self.conv1(x)
+        x = self.conv2(x)
+
+        Hx = self.Hattn(x)
+        Wx = self.Wattn(x)
+
+        x = self.conv3(Hx + Wx)
+        return x
+
+class PAA_e(nn.Module):
+    def __init__(self, in_channel, out_channel, base_size=None, stage=None):
+        super(PAA_e, self).__init__()
+        self.relu = nn.ReLU(True)
+        if base_size is not None and stage is not None:
+            self.stage_size = (base_size[0] // (2 ** stage), base_size[1] // (2 ** stage))
+        else:
+            self.stage_size = None
+
+        self.branch0 = Conv2d(in_channel, out_channel, 1)
+        self.branch1 = PAA_kernel(in_channel, out_channel, 3, self.stage_size)
+        self.branch2 = PAA_kernel(in_channel, out_channel, 5, self.stage_size)
+        self.branch3 = PAA_kernel(in_channel, out_channel, 7, self.stage_size)
+
+        self.conv_cat = Conv2d(4 * out_channel, out_channel, 3)
+        self.conv_res = Conv2d(in_channel, out_channel, 1)
+
+    def forward(self, x):
+        x0 = self.branch0(x)
+        x1 = self.branch1(x)
+        x2 = self.branch2(x)
+        x3 = self.branch3(x)
+
+        x_cat = self.conv_cat(torch.cat((x0, x1, x2, x3), 1))
+        x = self.relu(x_cat + self.conv_res(x))
+
+        return x

lib/modules/decoder_module.py

@@ -0,0 +1,44 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .layers import *
+class PAA_d(nn.Module):
+    def __init__(self, in_channel, out_channel=1, depth=64, base_size=None, stage=None):
+        super(PAA_d, self).__init__()
+        self.conv1 = Conv2d(in_channel ,depth, 3)
+        self.conv2 = Conv2d(depth, depth, 3)
+        self.conv3 = Conv2d(depth, depth, 3)
+        self.conv4 = Conv2d(depth, depth, 3)
+        self.conv5 = Conv2d(depth, out_channel, 3, bn=False)
+        
+        self.base_size = base_size
+        self.stage = stage
+        
+        if base_size is not None and stage is not None:
+            self.stage_size = (base_size[0] // (2 ** stage), base_size[1] // (2 ** stage))
+        else:
+            self.stage_size = [None, None]
+
+        self.Hattn = SelfAttention(depth, 'h', self.stage_size[0])
+        self.Wattn = SelfAttention(depth, 'w', self.stage_size[1])
+
+        self.upsample = lambda img, size: F.interpolate(img, size=size, mode='bilinear', align_corners=True)
+
+    def forward(self, fs): #f3 f4 f5 -> f3 f2 f1
+        fx = fs[0]
+        for i in range(1, len(fs)):
+            fs[i] = self.upsample(fs[i], fx.shape[-2:])
+        fx = torch.cat(fs[::-1], dim=1)
+
+        fx = self.conv1(fx)
+
+        Hfx = self.Hattn(fx)
+        Wfx = self.Wattn(fx)
+
+        fx = self.conv2(Hfx + Wfx)
+        fx = self.conv3(fx)
+        fx = self.conv4(fx)
+        out = self.conv5(fx)
+
+        return fx, out

lib/modules/layers.py

@@ -0,0 +1,175 @@
+from optparse import Option
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import cv2
+import numpy as np
+
+from kornia.morphology import dilation, erosion
+from torch.nn.parameter import Parameter
+from typing import Optional
+class ImagePyramid:
+    def __init__(self, ksize=7, sigma=1, channels=1):
+        self.ksize = ksize
+        self.sigma = sigma
+        self.channels = channels
+
+        k = cv2.getGaussianKernel(ksize, sigma)
+        k = np.outer(k, k)
+        k = torch.tensor(k).float()
+        self.kernel = k.repeat(channels, 1, 1, 1)
+        
+    def to(self, device):
+        self.kernel = self.kernel.to(device)
+        return self
+        
+    def cuda(self, idx=None):
+        if idx is None:
+            idx = torch.cuda.current_device()
+            
+        self.to(device="cuda:{}".format(idx))
+        return self
+
+    def expand(self, x):
+        z = torch.zeros_like(x)
+        x = torch.cat([x, z, z, z], dim=1)
+        x = F.pixel_shuffle(x, 2)
+        x = F.pad(x, (self.ksize // 2, ) * 4, mode='reflect')
+        x = F.conv2d(x, self.kernel * 4, groups=self.channels)
+        return x
+
+    def reduce(self, x):
+        x = F.pad(x, (self.ksize // 2, ) * 4, mode='reflect')
+        x = F.conv2d(x, self.kernel, groups=self.channels)
+        x = x[:, :, ::2, ::2]
+        return x
+
+    def deconstruct(self, x):
+        reduced_x = self.reduce(x)
+        expanded_reduced_x = self.expand(reduced_x)
+
+        if x.shape != expanded_reduced_x.shape:
+            expanded_reduced_x = F.interpolate(expanded_reduced_x, x.shape[-2:])
+
+        laplacian_x = x - expanded_reduced_x
+        return reduced_x, laplacian_x
+
+    def reconstruct(self, x, laplacian_x):
+        expanded_x = self.expand(x)
+        if laplacian_x.shape != expanded_x:
+            laplacian_x = F.interpolate(laplacian_x, expanded_x.shape[-2:], mode='bilinear', align_corners=True)
+        return expanded_x + laplacian_x
+
+class Transition:
+    def __init__(self, k=3):
+        self.kernel = torch.tensor(cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))).float()
+        
+    def to(self, device):
+        self.kernel = self.kernel.to(device)
+        return self
+        
+    def cuda(self, idx=None):
+        if idx is None:
+            idx = torch.cuda.current_device()
+            
+        self.to(device="cuda:{}".format(idx))
+        return self
+        
+    def __call__(self, x):
+        x = torch.sigmoid(x)
+        dx = dilation(x, self.kernel)
+        ex = erosion(x, self.kernel)
+        
+        return ((dx - ex) > .5).float()
+
+class Conv2d(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, padding='same', bias=False, bn=True, relu=False):
+        super(Conv2d, self).__init__()
+        if '__iter__' not in dir(kernel_size):
+            kernel_size = (kernel_size, kernel_size)
+        if '__iter__' not in dir(stride):
+            stride = (stride, stride)
+        if '__iter__' not in dir(dilation):
+            dilation = (dilation, dilation)
+
+        if padding == 'same':
+            width_pad_size = kernel_size[0] + (kernel_size[0] - 1) * (dilation[0] - 1)
+            height_pad_size = kernel_size[1] + (kernel_size[1] - 1) * (dilation[1] - 1)
+        elif padding == 'valid':
+            width_pad_size = 0
+            height_pad_size = 0
+        else:
+            if '__iter__' in dir(padding):
+                width_pad_size = padding[0] * 2
+                height_pad_size = padding[1] * 2
+            else:
+                width_pad_size = padding * 2
+                height_pad_size = padding * 2
+
+        width_pad_size = width_pad_size // 2 + (width_pad_size % 2 - 1)
+        height_pad_size = height_pad_size // 2 + (height_pad_size % 2 - 1)
+        pad_size = (width_pad_size, height_pad_size)
+        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, pad_size, dilation, groups, bias=bias)
+        self.reset_parameters()
+
+        if bn is True:
+            self.bn = nn.BatchNorm2d(out_channels)
+        else:
+            self.bn = None
+        
+        if relu is True:
+            self.relu = nn.ReLU(inplace=True)
+        else:
+            self.relu = None
+
+    def forward(self, x):
+        x = self.conv(x)
+        if self.bn is not None:
+            x = self.bn(x)
+        if self.relu is not None:
+            x = self.relu(x)
+        return x
+
+    def reset_parameters(self):
+        nn.init.kaiming_normal_(self.conv.weight)
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, in_channels, mode='hw', stage_size=None):
+        super(SelfAttention, self).__init__()
+
+        self.mode = mode
+
+        self.query_conv = Conv2d(in_channels, in_channels // 8, kernel_size=(1, 1))
+        self.key_conv = Conv2d(in_channels, in_channels // 8, kernel_size=(1, 1))
+        self.value_conv = Conv2d(in_channels, in_channels, kernel_size=(1, 1))
+
+        self.gamma = Parameter(torch.zeros(1))
+        self.softmax = nn.Softmax(dim=-1)
+        
+        self.stage_size = stage_size
+
+    def forward(self, x):
+        batch_size, channel, height, width = x.size()
+
+        axis = 1
+        if 'h' in self.mode:
+            axis *= height
+        if 'w' in self.mode:
+            axis *= width
+
+        view = (batch_size, -1, axis)
+
+        projected_query = self.query_conv(x).view(*view).permute(0, 2, 1)
+        projected_key = self.key_conv(x).view(*view)
+
+        attention_map = torch.bmm(projected_query, projected_key)
+        attention = self.softmax(attention_map)
+        projected_value = self.value_conv(x).view(*view)
+
+        out = torch.bmm(projected_value, attention.permute(0, 2, 1))
+        out = out.view(batch_size, channel, height, width)
+
+        out = self.gamma * out + x
+        return out

lib/optim/__init__.py

@@ -0,0 +1,2 @@
+from .losses import *
+from .scheduler import *

lib/optim/losses.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn.functional as F
+
+def bce_loss(pred, mask, reduction='none'):
+    bce = F.binary_cross_entropy(pred, mask, reduction=reduction)
+    return bce
+
+def weighted_bce_loss(pred, mask, reduction='none'):
+    weight = 1 + 5 * torch.abs(F.avg_pool2d(mask, kernel_size=31, stride=1, padding=15) - mask)
+    weight = weight.flatten()
+    
+    bce = weight * bce_loss(pred, mask, reduction='none').flatten()
+    
+    if reduction == 'mean':
+        bce = bce.mean()
+    
+    return bce
+
+def iou_loss(pred, mask, reduction='none'):
+    inter = pred * mask
+    union = pred + mask
+    iou = 1 - (inter + 1) / (union - inter + 1)
+
+    if reduction == 'mean':
+        iou = iou.mean()
+
+    return iou
+
+def bce_loss_with_logits(pred, mask, reduction='none'):
+    return bce_loss(torch.sigmoid(pred), mask, reduction=reduction)
+
+def weighted_bce_loss_with_logits(pred, mask, reduction='none'):
+    return weighted_bce_loss(torch.sigmoid(pred), mask, reduction=reduction)
+
+def iou_loss_with_logits(pred, mask, reduction='none'):
+    return iou_loss(torch.sigmoid(pred), mask, reduction=reduction)

lib/optim/scheduler.py

@@ -0,0 +1,30 @@
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class PolyLr(_LRScheduler):
+    def __init__(self, optimizer, gamma, max_iteration, minimum_lr=0, warmup_iteration=0, last_epoch=-1):
+        self.gamma = gamma
+        self.max_iteration = max_iteration
+        self.minimum_lr = minimum_lr
+        self.warmup_iteration = warmup_iteration
+        
+        self.last_epoch = None
+        self.base_lrs = []
+
+        super(PolyLr, self).__init__(optimizer, last_epoch)
+
+    def poly_lr(self, base_lr, step):
+        return (base_lr - self.minimum_lr) * ((1 - (step / self.max_iteration)) ** self.gamma) + self.minimum_lr
+
+    def warmup_lr(self, base_lr, alpha):
+        return base_lr * (1 / 10.0 * (1 - alpha) + alpha)
+
+    def get_lr(self):
+        if self.last_epoch < self.warmup_iteration:
+            alpha = self.last_epoch / self.warmup_iteration
+            lrs = [min(self.warmup_lr(base_lr, alpha), self.poly_lr(base_lr, self.last_epoch)) for base_lr in
+                    self.base_lrs]
+        else:
+            lrs = [self.poly_lr(base_lr, self.last_epoch) for base_lr in self.base_lrs]
+
+        return lrs

requirements.txt

@@ -0,0 +1,9 @@
+easydict
+timm
+tqdm
+kornia
+numpy
+opencv-python
+pyyaml
+pandas
+scipy

run/Eval.py

@@ -0,0 +1,138 @@
+import os
+import sys
+import tqdm
+
+import pandas as pd
+import numpy as np
+
+from PIL import Image
+
+filepath = os.path.split(os.path.abspath(__file__))[0]
+repopath = os.path.split(filepath)[0]
+sys.path.append(repopath)
+
+from utils.eval_functions import *
+from utils.misc import *
+
+BETA = 1.0
+
+def evaluate(opt, args):
+    if os.path.isdir(opt.Eval.result_path) is False:
+        os.makedirs(opt.Eval.result_path)
+        
+    method = os.path.split(opt.Eval.pred_root)[-1]
+
+    if args.verbose is True:
+        print('#' * 20, 'Start Evaluation', '#' * 20)
+        datasets = tqdm.tqdm(opt.Eval.datasets, desc='Expr - ' + method, total=len(
+            opt.Eval.datasets), position=0, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}')
+    else:
+        datasets = opt.Eval.datasets
+
+    results = []
+
+    for dataset in datasets:
+        pred_root = os.path.join(opt.Eval.pred_root, dataset)
+        gt_root = os.path.join(opt.Eval.gt_root, dataset, 'masks')
+
+        preds = os.listdir(pred_root)
+        gts = os.listdir(gt_root)
+        
+        preds = sort(preds)
+        gts = sort(gts)
+        
+        preds = [i for i in preds if i in gts]
+        gts = [i for i in gts if i in preds]
+        
+        FM = Fmeasure()
+        WFM = WeightedFmeasure()
+        SM = Smeasure()
+        EM = Emeasure()
+        MAE = Mae()
+        MSE = Mse()
+        MBA = BoundaryAccuracy()
+        IOU = IoU()
+        BIOU = BIoU()
+        TIOU = TIoU()
+
+        if args.verbose is True:
+            samples = tqdm.tqdm(enumerate(zip(preds, gts)), desc=dataset + ' - Evaluation', total=len(
+                preds), position=1, leave=False, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}')
+        else:
+            samples = enumerate(zip(preds, gts))
+
+        for i, sample in samples:
+            pred, gt = sample
+
+            pred_mask = np.array(Image.open(os.path.join(pred_root, pred)).convert('L'))
+            gt_mask = np.array(Image.open(os.path.join(gt_root, gt)).convert('L'))
+
+            if len(pred_mask.shape) != 2:
+                pred_mask = pred_mask[:, :, 0]
+            if len(gt_mask.shape) != 2:
+                gt_mask = gt_mask[:, :, 0]
+
+            assert pred_mask.shape == gt_mask.shape, print(pred, 'does not match the size of', gt)
+            # print(gt_mask.max())
+
+            FM.step( pred=pred_mask, gt=gt_mask)
+            WFM.step(pred=pred_mask, gt=gt_mask)
+            SM.step( pred=pred_mask, gt=gt_mask)
+            EM.step( pred=pred_mask, gt=gt_mask)
+            MAE.step(pred=pred_mask, gt=gt_mask)
+            MSE.step(pred=pred_mask, gt=gt_mask)
+            MBA.step(pred=pred_mask, gt=gt_mask)
+            IOU.step(pred=pred_mask, gt=gt_mask)
+            BIOU.step(pred=pred_mask, gt=gt_mask)
+            TIOU.step(pred=pred_mask, gt=gt_mask)
+            
+        result = []
+
+        Sm =  SM.get_results()["sm"]
+        wFm = WFM.get_results()["wfm"]
+        mae = MAE.get_results()["mae"]
+        mse = MSE.get_results()["mse"]
+        mBA = MBA.get_results()["mba"]
+        
+        Fm =  FM.get_results()["fm"]
+        Em =  EM.get_results()["em"]
+        Iou = IOU.get_results()["iou"]
+        BIou = BIOU.get_results()["biou"]
+        TIou = TIOU.get_results()["tiou"]
+        
+        adpEm = Em["adp"]
+        avgEm = Em["curve"].mean()
+        maxEm = Em["curve"].max()
+        adpFm = Fm["adp"]
+        avgFm = Fm["curve"].mean()
+        maxFm = Fm["curve"].max()
+        avgIou = Iou["curve"].mean()
+        maxIou = Iou["curve"].max()
+        avgBIou = BIou["curve"].mean()
+        maxBIou = BIou["curve"].max()
+        avgTIou = TIou["curve"].mean()
+        maxTIou = TIou["curve"].max()
+        
+        out = dict()
+        for metric in opt.Eval.metrics:
+            out[metric] = eval(metric)
+
+        pkl = os.path.join(opt.Eval.result_path, 'result_' + dataset + '.pkl')
+        if os.path.isfile(pkl) is True:
+            result = pd.read_pickle(pkl)
+            result.loc[method] = out
+            result.to_pickle(pkl)
+        else:
+            result = pd.DataFrame(data=out, index=[method])
+            result.to_pickle(pkl)
+        result.to_csv(os.path.join(opt.Eval.result_path, 'result_' + dataset + '.csv'))
+        results.append(result)
+        
+    if args.verbose is True:
+        for dataset, result in zip(datasets, results):
+            print('###', dataset, '###', '\n', result.sort_index(), '\n')
+    
+if __name__ == "__main__":
+    args = parse_args()
+    opt = load_config(args.config)
+    evaluate(opt, args)

run/Inference.py

@@ -0,0 +1,167 @@
+import os
+import cv2
+import sys
+import tqdm
+import torch
+import argparse
+
+import numpy as np
+
+from PIL import Image
+
+filepath = os.path.split(os.path.abspath(__file__))[0]
+repopath = os.path.split(filepath)[0]
+sys.path.append(repopath)
+
+from lib import *
+from utils.misc import *
+from data.dataloader import *
+from data.custom_transforms import *
+
+torch.backends.cuda.matmul.allow_tf32 = False
+torch.backends.cudnn.allow_tf32 = False
+
+def _args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', '-c',     type=str,            default='configs/InSPyReNet_SwinB.yaml')
+    parser.add_argument('--source', '-s',     type=str)
+    parser.add_argument('--dest', '-d',       type=str,            default=None)
+    parser.add_argument('--type', '-t',       type=str,            default='map')
+    parser.add_argument('--gpu', '-g',        action='store_true', default=False)
+    parser.add_argument('--jit', '-j',        action='store_true', default=False)
+    parser.add_argument('--verbose', '-v',    action='store_true', default=False)
+    return parser.parse_args()
+
+def get_format(source):
+    img_count = len([i for i in source if i.lower().endswith(('.jpg', '.png', '.jpeg'))])
+    vid_count = len([i for i in source if i.lower().endswith(('.mp4', '.avi', '.mov' ))])
+    
+    if img_count * vid_count != 0:
+        return ''
+    elif img_count != 0:
+        return 'Image'
+    elif vid_count != 0:
+        return 'Video'
+    else:
+        return ''
+
+def inference(opt, args):
+    model = eval(opt.Model.name)(**opt.Model)
+    model.load_state_dict(torch.load(os.path.join(
+        opt.Test.Checkpoint.checkpoint_dir, 'latest.pth'), map_location=torch.device('cpu')), strict=True)
+    
+    if args.gpu is True:
+        model = model.cuda()
+    model.eval()
+        
+    if args.jit is True:
+        if os.path.isfile(os.path.join(opt.Test.Checkpoint.checkpoint_dir, 'jit.pt')) is False:
+            model = Simplify(model)
+            model = torch.jit.trace(model, torch.rand(1, 3, *opt.Test.Dataset.transforms.static_resize.size).cuda(), strict=False)
+            torch.jit.save(model, os.path.join(opt.Test.Checkpoint.checkpoint_dir, 'jit.pt'))
+        
+        else:
+            del model
+            model = torch.jit.load(os.path.join(opt.Test.Checkpoint.checkpoint_dir, 'jit.pt'))
+                
+    save_dir = None
+    _format = None
+    
+    if args.source.isnumeric() is True:
+        _format = 'Webcam'
+
+    elif os.path.isdir(args.source):
+        save_dir = os.path.join('results', args.source.split(os.sep)[-1])
+        _format = get_format(os.listdir(args.source))
+
+    elif os.path.isfile(args.source):
+        save_dir = 'results'
+        _format = get_format([args.source])
+        
+    if args.dest is not None:
+        save_dir = args.dest
+        
+    if save_dir is not None:
+        os.makedirs(save_dir, exist_ok=True)
+    
+    sample_list = eval(_format + 'Loader')(args.source, opt.Test.Dataset.transforms)
+
+    if args.verbose is True:
+        samples = tqdm.tqdm(sample_list, desc='Inference', total=len(
+            sample_list), position=0, leave=False, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}')
+    else:
+        samples = sample_list
+        
+    writer = None
+    background = None
+
+    for sample in samples:
+        if _format == 'Video' and writer is None:
+            writer = cv2.VideoWriter(os.path.join(save_dir, sample['name'] + '.mp4'), cv2.VideoWriter_fourcc(*'mp4v'), sample_list.fps, sample['shape'][::-1])
+            samples.total += int(sample_list.cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        if _format == 'Video' and sample['image'] is None:
+            if writer is not None:
+                writer.release()
+            writer = None
+            continue
+        
+        if args.gpu is True:
+            sample = to_cuda(sample)
+
+        with torch.no_grad():
+            if args.jit is True:
+                out = model(sample['image'])
+            else:
+                out = model(sample)
+                
+                    
+        pred = to_numpy(out['pred'], sample['shape'])
+        img = np.array(sample['original'])
+        
+        if args.type == 'map':
+            img = (np.stack([pred] * 3, axis=-1) * 255).astype(np.uint8)
+        elif args.type == 'rgba':
+            r, g, b = cv2.split(img)
+            pred = (pred * 255).astype(np.uint8)
+            img = cv2.merge([r, g, b, pred])
+        elif args.type == 'green':
+            bg = np.stack([np.ones_like(pred)] * 3, axis=-1) * [120, 255, 155]
+            img = img * pred[..., np.newaxis] + bg * (1 - pred[..., np.newaxis])
+        elif args.type == 'blur':
+            img = img * pred[..., np.newaxis] + cv2.GaussianBlur(img, (0, 0), 15) * (1 - pred[..., np.newaxis])
+        elif args.type == 'overlay':
+            bg = (np.stack([np.ones_like(pred)] * 3, axis=-1) * [120, 255, 155] + img) // 2
+            img = bg * pred[..., np.newaxis] + img * (1 - pred[..., np.newaxis])
+            border = cv2.Canny(((pred > .5) * 255).astype(np.uint8), 50, 100)
+            img[border != 0] = [120, 255, 155]
+        elif args.type.lower().endswith(('.jpg', '.jpeg', '.png')):
+            if background is None:
+                background = cv2.cvtColor(cv2.imread(args.type), cv2.COLOR_BGR2RGB)
+                background = cv2.resize(background, img.shape[:2][::-1])
+            img = img * pred[..., np.newaxis] + background * (1 - pred[..., np.newaxis])
+        elif args.type == 'debug':
+            debs = []
+            for k in opt.Train.Debug.keys:
+                debs.extend(out[k])
+            for i, j in enumerate(debs):
+                log = torch.sigmoid(j).cpu().detach().numpy().squeeze()
+                log = ((log - log.min()) / (log.max() - log.min()) * 255).astype(np.uint8)
+                log = cv2.cvtColor(log, cv2.COLOR_GRAY2RGB)
+                log = cv2.resize(log, img.shape[:2][::-1])
+                Image.fromarray(log).save(os.path.join(save_dir, sample['name'] + '_' + str(i) + '.png'))    
+                # size=img.shape[:2][::-1]
+            
+            
+        img = img.astype(np.uint8)
+        
+        if _format == 'Image':
+            Image.fromarray(img).save(os.path.join(save_dir, sample['name'] + '.png'))
+        elif _format == 'Video' and writer is not None:
+            writer.write(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+        elif _format == 'Webcam':
+            cv2.imshow('InSPyReNet', img)
+
+if __name__ == "__main__":
+    args = _args()
+    opt = load_config(args.config)
+    inference(opt, args)

run/Test.py

@@ -0,0 +1,59 @@
+import os
+import sys
+import tqdm
+import torch
+
+import numpy as np
+
+from PIL import Image
+from torch.utils.data.dataloader import DataLoader
+
+filepath = os.path.split(os.path.abspath(__file__))[0]
+repopath = os.path.split(filepath)[0]
+sys.path.append(repopath)
+
+from lib import *
+from utils.misc import *
+from data.dataloader import *
+
+torch.backends.cuda.matmul.allow_tf32 = False
+torch.backends.cudnn.allow_tf32 = False
+
+def test(opt, args):
+    model = eval(opt.Model.name)(**opt.Model)
+    model.load_state_dict(torch.load(os.path.join(opt.Test.Checkpoint.checkpoint_dir, 'latest.pth')), strict=True)
+        
+    model.cuda()
+    model.eval()
+
+    if args.verbose is True:
+        sets = tqdm.tqdm(opt.Test.Dataset.sets, desc='Total TestSet', total=len(
+            opt.Test.Dataset.sets), position=0, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}')
+    else:
+        sets = opt.Test.Dataset.sets
+
+    for set in sets:
+        save_path = os.path.join(opt.Test.Checkpoint.checkpoint_dir, set)
+
+        os.makedirs(save_path, exist_ok=True)
+        test_dataset = eval(opt.Test.Dataset.type)(opt.Test.Dataset.root, [set], opt.Test.Dataset.transforms)
+        test_loader  = DataLoader(dataset=test_dataset, batch_size=1, num_workers=opt.Test.Dataloader.num_workers, pin_memory=opt.Test.Dataloader.pin_memory)
+
+        if args.verbose is True:
+            samples = tqdm.tqdm(test_loader, desc=set + ' - Test', total=len(test_loader),
+                                position=1, leave=False, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}')
+        else:
+            samples = test_loader
+
+        for sample in samples:
+            sample = to_cuda(sample)
+            with torch.no_grad():
+                out = model(sample)
+            
+            pred = to_numpy(out['pred'], sample['shape'])
+            Image.fromarray((pred * 255).astype(np.uint8)).save(os.path.join(save_path, sample['name'][0] + '.png'))
+
+if __name__ == "__main__":
+    args = parse_args()
+    opt = load_config(args.config)
+    test(opt, args)