From 6c1df067ce500399684afe64800169c6e5423310 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 11 Oct 2023 08:39:59 +0200 Subject: [PATCH 01/11] flaky tests --- tests/unittests/utilities/test_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/utilities/test_plot.py b/tests/unittests/utilities/test_plot.py index f28a6444e6f..9d536766b54 100644 --- a/tests/unittests/utilities/test_plot.py +++ b/tests/unittests/utilities/test_plot.py @@ -609,7 +609,7 @@ pytest.param(BLEUScore, _text_input_3, _text_input_4, id="bleu score"), pytest.param(CHRFScore, _text_input_3, _text_input_4, id="bleu score"), pytest.param( - partial(InfoLM, model_name_or_path="google/bert_uncased_L-2_H-128_A-2", idf=False), + partial(InfoLM, model_name_or_path="google/bert_uncased_L-2_H-128_A-2", idf=False, verbose=False), _text_input_1, _text_input_2, id="info lm", From e260deec94653a3bd3b1f904305560e776fc7c04 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 11 Oct 2023 08:46:12 +0200 Subject: [PATCH 02/11] increase timeout --- .azure/gpu-unittests.yml | 2 +- src/torchmetrics/functional/multimodal/clip_iqa.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index b22d2ae5191..9f6c0e82348 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -110,7 +110,7 @@ jobs: - bash: python -m pytest torchmetrics --timeout=240 --durations=50 env: - DOCTEST_DOWNLOAD_TIMEOUT: "120" + DOCTEST_DOWNLOAD_TIMEOUT: "240" SKIP_SLOW_DOCTEST: "1" workingDirectory: src displayName: "DocTesting" diff --git a/src/torchmetrics/functional/multimodal/clip_iqa.py b/src/torchmetrics/functional/multimodal/clip_iqa.py index 4f5a6ccbb5f..4ad31ca937d 100644 --- a/src/torchmetrics/functional/multimodal/clip_iqa.py +++ b/src/torchmetrics/functional/multimodal/clip_iqa.py @@ -29,7 +29,7 @@ def _download_clip() -> None: _CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") if _SKIP_SLOW_DOCTEST and not _try_proceed_with_timeout(_download_clip): - __doctest_skip__ = ["clip_score"] + __doctest_skip__ = ["clip_image_quality_assessment"] else: __doctest_skip__ = ["clip_image_quality_assessment"] From ae39d13e468c5f907c5ada3aa3500ca81e36e150 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 11 Oct 2023 10:54:11 +0200 Subject: [PATCH 03/11] disable progress bar --- src/torchmetrics/image/fid.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/image/fid.py b/src/torchmetrics/image/fid.py index 2ba787cf017..4de3818430c 100644 --- a/src/torchmetrics/image/fid.py +++ b/src/torchmetrics/image/fid.py @@ -29,7 +29,12 @@ __doctest_skip__ += ["FrechetInceptionDistance.plot"] if _TORCH_FIDELITY_AVAILABLE: - from torch_fidelity.feature_extractor_inceptionv3 import FeatureExtractorInceptionV3 as _FeatureExtractorInceptionV3 + from torch_fidelity.feature_extractor_inceptionv3 import ( + URL_INCEPTION_V3, + ) + from torch_fidelity.feature_extractor_inceptionv3 import ( + FeatureExtractorInceptionV3 as _FeatureExtractorInceptionV3, + ) from torch_fidelity.helpers import vassert from torch_fidelity.interpolate_compat_tensorflow import interpolate_bilinear_2d_like_tensorflow1x else: @@ -60,8 +65,11 @@ def __init__( "NoTrainInceptionV3 module requires that `Torch-fidelity` is installed." " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." ) - + if feature_extractor_weights_path is None: + torch.hub.load_state_dict_from_url(URL_INCEPTION_V3, progress=False) + feature_extractor_weights_path = f"{torch.hub.get_dir()}/checkpoints/{URL_INCEPTION_V3.split('/')[-1]}" super().__init__(name, features_list, feature_extractor_weights_path) + # put into evaluation mode self.eval() From f09bcc841d140d0e872739c3cc1bac88f58ef1c3 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 11 Oct 2023 11:18:01 +0200 Subject: [PATCH 04/11] new link --- docs/source/links.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/links.rst b/docs/source/links.rst index c967c454d43..4ee095dd659 100644 --- a/docs/source/links.rst +++ b/docs/source/links.rst @@ -125,7 +125,7 @@ .. _SRMRpy: https://github.com/jfsantos/SRMRpy .. _Permutation invariant training: https://arxiv.org/abs/1607.00325 .. _ranking ref1: https://link.springer.com/chapter/10.1007/978-0-387-09823-4_34 -.. _Spectral Distortion Index: https://www.ingentaconnect.com/content/asprs/pers/2008/00000074/00000002/art00003;jsessionid=nzjnb3v9xxr1.x-ic-live-03 +.. _Spectral Distortion Index: https://www.semanticscholar.org/paper/Multispectral-and-panchromatic-data-fusion-without-Alparone-Aiazzi/b6db12e3785326577cb95fd743fecbf5bc66c7c9 .. _Relative dimensionless global error synthesis: https://ieeexplore.ieee.org/abstract/document/4317530 .. _fid ref1: https://arxiv.org/abs/1512.00567 .. _fid ref2: https://arxiv.org/abs/1706.08500 From da04e7cc9c8324cbb7f5ea1d652471ee1dcc989d Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 11 Oct 2023 13:18:42 +0200 Subject: [PATCH 05/11] move checking --- src/torchmetrics/image/kid.py | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/torchmetrics/image/kid.py b/src/torchmetrics/image/kid.py index cafa1464b6e..2edfbbb9f8c 100644 --- a/src/torchmetrics/image/kid.py +++ b/src/torchmetrics/image/kid.py @@ -184,24 +184,6 @@ def __init__( UserWarning, ) - if isinstance(feature, (str, int)): - if not _TORCH_FIDELITY_AVAILABLE: - raise ModuleNotFoundError( - "Kernel Inception Distance metric requires that `Torch-fidelity` is installed." - " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." - ) - valid_int_input = ("logits_unbiased", 64, 192, 768, 2048) - if feature not in valid_int_input: - raise ValueError( - f"Integer input to argument `feature` must be one of {valid_int_input}, but got {feature}." - ) - - self.inception: Module = NoTrainInceptionV3(name="inception-v3-compat", features_list=[str(feature)]) - elif isinstance(feature, Module): - self.inception = feature - else: - raise TypeError("Got unknown input to argument `feature`") - if not (isinstance(subsets, int) and subsets > 0): raise ValueError("Argument `subsets` expected to be integer larger than 0") self.subsets = subsets @@ -230,6 +212,24 @@ def __init__( raise ValueError("Argument `normalize` expected to be a bool") self.normalize = normalize + if isinstance(feature, (str, int)): + if not _TORCH_FIDELITY_AVAILABLE: + raise ModuleNotFoundError( + "Kernel Inception Distance metric requires that `Torch-fidelity` is installed." + " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." + ) + valid_int_input = ("logits_unbiased", 64, 192, 768, 2048) + if feature not in valid_int_input: + raise ValueError( + f"Integer input to argument `feature` must be one of {valid_int_input}, but got {feature}." + ) + + self.inception: Module = NoTrainInceptionV3(name="inception-v3-compat", features_list=[str(feature)]) + elif isinstance(feature, Module): + self.inception = feature + else: + raise TypeError("Got unknown input to argument `feature`") + # states for extracted features self.add_state("real_features", [], dist_reduce_fx=None) self.add_state("fake_features", [], dist_reduce_fx=None) From 45ec91e13c7fddddb56bc661c068097ee237aa8a Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:33:41 +0200 Subject: [PATCH 06/11] try something --- .azure/gpu-unittests.yml | 2 +- .github/workflows/docker-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 2227386c2a7..03b22ffa905 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -19,7 +19,7 @@ jobs: matrix: "PyTorch | old": # Torch does not have build wheels with old Torch versions for newer CUDA - docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.8.0-py3.9-torch1.10" + docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.3.0-py3.9-torch1.10" agent-pool: "lit-rtx-3090" torch-ver: "1.10.2" "PyTorch | 1.X": diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3e6610521dc..374c046778f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -66,7 +66,7 @@ jobs: include: # These are the base images for PL release docker images, # so include at least all of the combinations in release-dockers.yml. - - { python: "3.9", pytorch: "1.10", cuda: "11.8.0", ubuntu: "22.04" } + - { python: "3.9", pytorch: "1.10", cuda: "11.3.0", ubuntu: "22.04" } - { python: "3.9", pytorch: "1.11", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.9", pytorch: "1.13", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.10", pytorch: "2.0", cuda: "11.8.0", ubuntu: "22.04" } From 1949fcc62aa63f77343ecf05852b4da6b5940e60 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:36:46 +0200 Subject: [PATCH 07/11] revert some changes --- .../functional/multimodal/clip_iqa.py | 2 +- src/torchmetrics/image/fid.py | 12 ++----- src/torchmetrics/image/kid.py | 36 +++++++++---------- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/torchmetrics/functional/multimodal/clip_iqa.py b/src/torchmetrics/functional/multimodal/clip_iqa.py index 4ad31ca937d..4f5a6ccbb5f 100644 --- a/src/torchmetrics/functional/multimodal/clip_iqa.py +++ b/src/torchmetrics/functional/multimodal/clip_iqa.py @@ -29,7 +29,7 @@ def _download_clip() -> None: _CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") if _SKIP_SLOW_DOCTEST and not _try_proceed_with_timeout(_download_clip): - __doctest_skip__ = ["clip_image_quality_assessment"] + __doctest_skip__ = ["clip_score"] else: __doctest_skip__ = ["clip_image_quality_assessment"] diff --git a/src/torchmetrics/image/fid.py b/src/torchmetrics/image/fid.py index 72b2c31869d..bc830fdd0b7 100644 --- a/src/torchmetrics/image/fid.py +++ b/src/torchmetrics/image/fid.py @@ -27,12 +27,7 @@ __doctest_skip__ = ["FrechetInceptionDistance.plot"] if _TORCH_FIDELITY_AVAILABLE: - from torch_fidelity.feature_extractor_inceptionv3 import ( - URL_INCEPTION_V3, - ) - from torch_fidelity.feature_extractor_inceptionv3 import ( - FeatureExtractorInceptionV3 as _FeatureExtractorInceptionV3, - ) + from torch_fidelity.feature_extractor_inceptionv3 import FeatureExtractorInceptionV3 as _FeatureExtractorInceptionV3 from torch_fidelity.helpers import vassert from torch_fidelity.interpolate_compat_tensorflow import interpolate_bilinear_2d_like_tensorflow1x else: @@ -60,11 +55,8 @@ def __init__( "NoTrainInceptionV3 module requires that `Torch-fidelity` is installed." " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." ) - if feature_extractor_weights_path is None: - torch.hub.load_state_dict_from_url(URL_INCEPTION_V3, progress=False) - feature_extractor_weights_path = f"{torch.hub.get_dir()}/checkpoints/{URL_INCEPTION_V3.split('/')[-1]}" - super().__init__(name, features_list, feature_extractor_weights_path) + super().__init__(name, features_list, feature_extractor_weights_path) # put into evaluation mode self.eval() diff --git a/src/torchmetrics/image/kid.py b/src/torchmetrics/image/kid.py index 2edfbbb9f8c..cafa1464b6e 100644 --- a/src/torchmetrics/image/kid.py +++ b/src/torchmetrics/image/kid.py @@ -184,6 +184,24 @@ def __init__( UserWarning, ) + if isinstance(feature, (str, int)): + if not _TORCH_FIDELITY_AVAILABLE: + raise ModuleNotFoundError( + "Kernel Inception Distance metric requires that `Torch-fidelity` is installed." + " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." + ) + valid_int_input = ("logits_unbiased", 64, 192, 768, 2048) + if feature not in valid_int_input: + raise ValueError( + f"Integer input to argument `feature` must be one of {valid_int_input}, but got {feature}." + ) + + self.inception: Module = NoTrainInceptionV3(name="inception-v3-compat", features_list=[str(feature)]) + elif isinstance(feature, Module): + self.inception = feature + else: + raise TypeError("Got unknown input to argument `feature`") + if not (isinstance(subsets, int) and subsets > 0): raise ValueError("Argument `subsets` expected to be integer larger than 0") self.subsets = subsets @@ -212,24 +230,6 @@ def __init__( raise ValueError("Argument `normalize` expected to be a bool") self.normalize = normalize - if isinstance(feature, (str, int)): - if not _TORCH_FIDELITY_AVAILABLE: - raise ModuleNotFoundError( - "Kernel Inception Distance metric requires that `Torch-fidelity` is installed." - " Either install as `pip install torchmetrics[image]` or `pip install torch-fidelity`." - ) - valid_int_input = ("logits_unbiased", 64, 192, 768, 2048) - if feature not in valid_int_input: - raise ValueError( - f"Integer input to argument `feature` must be one of {valid_int_input}, but got {feature}." - ) - - self.inception: Module = NoTrainInceptionV3(name="inception-v3-compat", features_list=[str(feature)]) - elif isinstance(feature, Module): - self.inception = feature - else: - raise TypeError("Got unknown input to argument `feature`") - # states for extracted features self.add_state("real_features", [], dist_reduce_fx=None) self.add_state("fake_features", [], dist_reduce_fx=None) From 5eb916c976f096cefbe9f4a810c0a0f64c4d7fd4 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:40:03 +0200 Subject: [PATCH 08/11] new docker --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 374c046778f..9cbd546745d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -66,7 +66,7 @@ jobs: include: # These are the base images for PL release docker images, # so include at least all of the combinations in release-dockers.yml. - - { python: "3.9", pytorch: "1.10", cuda: "11.3.0", ubuntu: "22.04" } + - { python: "3.9", pytorch: "1.10", cuda: "11.3.1", ubuntu: "22.04" } - { python: "3.9", pytorch: "1.11", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.9", pytorch: "1.13", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.10", pytorch: "2.0", cuda: "11.8.0", ubuntu: "22.04" } From 07b3193ce593af68e375a2cf7a140f334405b4ab Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:41:14 +0200 Subject: [PATCH 09/11] exclude windows 2.1 --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b73982287b9..5ec4c3dad76 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -46,7 +46,7 @@ jobs: - { os: "windows-2022", python-version: "3.8", pytorch-version: "1.13.1" } - { os: "windows-2022", python-version: "3.9", pytorch-version: "1.13.1" } - { os: "windows-2022", python-version: "3.10", pytorch-version: "2.0.1" } - - { os: "windows-2022", python-version: "3.11", pytorch-version: "2.1.0" } + # - { os: "windows-2022", python-version: "3.11", pytorch-version: "2.1.0" } # TODO env: PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} From 85019d761bd64b50017f25cd539ff784879cdf70 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:44:09 +0200 Subject: [PATCH 10/11] correct ubuntu version --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9cbd546745d..ae28feccdd7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -66,7 +66,7 @@ jobs: include: # These are the base images for PL release docker images, # so include at least all of the combinations in release-dockers.yml. - - { python: "3.9", pytorch: "1.10", cuda: "11.3.1", ubuntu: "22.04" } + - { python: "3.9", pytorch: "1.10", cuda: "11.3.1", ubuntu: "20.04" } - { python: "3.9", pytorch: "1.11", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.9", pytorch: "1.13", cuda: "11.8.0", ubuntu: "22.04" } - { python: "3.10", pytorch: "2.0", cuda: "11.8.0", ubuntu: "22.04" } From 6167e5dab87514bdffba54c7550c96367d011e34 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 17 Oct 2023 13:52:26 +0200 Subject: [PATCH 11/11] correct image in azure --- .azure/gpu-unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 03b22ffa905..1f0b5fcad61 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -19,7 +19,7 @@ jobs: matrix: "PyTorch | old": # Torch does not have build wheels with old Torch versions for newer CUDA - docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.3.0-py3.9-torch1.10" + docker-image: "pytorchlightning/torchmetrics:ubuntu20.04-cuda11.3.1-py3.9-torch1.10" agent-pool: "lit-rtx-3090" torch-ver: "1.10.2" "PyTorch | 1.X":