From 3c3596cd6bd3b24e1626dde2030a49c4eebd3125 Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Wed, 9 Oct 2024 19:03:12 -0700 Subject: [PATCH 1/8] add virchow and ciomedclip model assets --- .../asset.yaml | 4 + .../description.md | 75 +++++++++++++ .../model.yaml | 8 ++ .../spec.yaml | 34 ++++++ assets/models/system/Virchow/asset.yaml | 4 + assets/models/system/Virchow/description.md | 102 +++++++++++++++++ assets/models/system/Virchow/model.yaml | 8 ++ assets/models/system/Virchow/spec.yaml | 34 ++++++ assets/models/system/Virchow2/asset.yaml | 4 + assets/models/system/Virchow2/description.md | 105 ++++++++++++++++++ assets/models/system/Virchow2/model.yaml | 8 ++ assets/models/system/Virchow2/spec.yaml | 34 ++++++ 12 files changed, 420 insertions(+) create mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml create mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md create mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml create mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml create mode 100644 assets/models/system/Virchow/asset.yaml create mode 100644 assets/models/system/Virchow/description.md create mode 100644 assets/models/system/Virchow/model.yaml create mode 100644 assets/models/system/Virchow/spec.yaml create mode 100644 assets/models/system/Virchow2/asset.yaml create mode 100644 assets/models/system/Virchow2/description.md create mode 100644 assets/models/system/Virchow2/model.yaml create mode 100644 assets/models/system/Virchow2/spec.yaml diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml new file mode 100644 index 0000000000..fcf5c5a05b --- /dev/null +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml @@ -0,0 +1,4 @@ +extra_config: model.yaml +spec: spec.yaml +type: model +categories: ["Foundation Models"] diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md new file mode 100644 index 0000000000..24d31af43d --- /dev/null +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md @@ -0,0 +1,75 @@ + +[BiomedCLIP](https://aka.ms/biomedclip-paper) is a biomedical vision-language foundation model that is pretrained on [PMC-15M](https://aka.ms/biomedclip-paper), a dataset of 15 million figure-caption pairs extracted from biomedical research articles in PubMed Central, using contrastive learning. +It uses PubMedBERT as the text encoder and Vision Transformer as the image encoder, with domain-specific adaptations. +It can perform various vision-language processing (VLP) tasks such as cross-modal retrieval, image classification, and visual question answering. + +## Citation + +```bibtex +@misc{https://doi.org/10.48550/arXiv.2303.00915, + doi = {10.48550/ARXIV.2303.00915}, + url = {https://arxiv.org/abs/2303.00915}, + author = {Zhang, Sheng and Xu, Yanbo and Usuyama, Naoto and Bagga, Jaspreet and Tinn, Robert and Preston, Sam and Rao, Rajesh and Wei, Mu and Valluri, Naveen and Wong, Cliff and Lungren, Matthew and Naumann, Tristan and Poon, Hoifung}, + title = {Large-Scale Domain-Specific Pretraining for Biomedical Vision-Language Processing}, + publisher = {arXiv}, + year = {2023}, +} +``` + +## Model Use + +### Sample Input + +```json +data = { + "input_data": { + "columns": [ + "image", + "text" + ], + "index":[0, 1, 2], + "data": [ + ["image1", "labe1, label2, label3"], + ["image2", "labe1, label2, label3"], + ["image3", "labe1, label2, label3"], + ] + } +} +``` +### Sample Output +```json +[ + { + "probs": [0.95, 0.03, 0.02], + "labels": ["label1", "label2", "label3"] + }, + { + "probs": [0.04, 0.93, 0.03], + "labels": ["label1", "label2", "label3"] + } +] +``` + +### Intended Use + +This model is intended to be used solely for (I) future research on visual-language processing and (II) reproducibility of the experimental results reported in the reference paper. + +#### Primary Intended Use + +The primary intended use is to support AI researchers building on top of this work. BiomedCLIP and its associated models should be helpful for exploring various biomedical VLP research questions, especially in the radiology domain. + +#### Out-of-Scope Use + +**Any** deployed use case of the model --- commercial or otherwise --- is currently out of scope. Although we evaluated the models using a broad set of publicly-available research benchmarks, the models and evaluations are not intended for deployed use cases. Please refer to [the associated paper](https://aka.ms/biomedclip-paper) for more details. + +## Data + +This model builds upon [PMC-15M dataset](https://aka.ms/biomedclip-paper), which is a large-scale parallel image-text dataset for biomedical vision-language processing. It contains 15 million figure-caption pairs extracted from biomedical research articles in PubMed Central. It covers a diverse range of biomedical image types, such as microscopy, radiography, histology, and more. + +## Limitations + +This model was developed using English corpora, and thus can be considered English-only. + +## Further information + +Please refer to the corresponding paper, ["Large-Scale Domain-Specific Pretraining for Biomedical Vision-Language Processing"](https://aka.ms/biomedclip-paper) for additional details on the model training and evaluation. \ No newline at end of file diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml new file mode 100644 index 0000000000..01fd42f578 --- /dev/null +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml @@ -0,0 +1,8 @@ +path: + container_name: models + container_path: huggingface/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/mlflow_model_folder + storage_name: automlcesdkdataresources + type: azureblob +publish: + description: description.md + type: mlflow_model diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml new file mode 100644 index 0000000000..0d0ce199b4 --- /dev/null +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml @@ -0,0 +1,34 @@ +$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json + +name: BiomedCLIP-PubMedBERT_256-vit_base_patch16_224 +path: ./ + +properties: + inference-min-sku-spec: 6|1|112|64 + inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 + languages: en + SharedComputeCapacityEnabled: true + +tags: + task: zero-shot-image-classification + industry: health-and-life-sciences + Preview: "" + inference_supported_envs: + - hf + license: mit + author: Microsoft + hiddenlayerscanned: "" + SharedComputeCapacityEnabled: "" + inference_compute_allow_list: + [ + Standard_NC6s_v3, + Standard_NC12s_v3, + Standard_NC24s_v3, + Standard_NC24ads_A100_v4, + Standard_NC48ads_A100_v4, + Standard_NC96ads_A100_v4, + Standard_ND96asr_v4, + Standard_ND96amsr_A100_v4, + Standard_ND40rs_v2, + ] +version: 1 \ No newline at end of file diff --git a/assets/models/system/Virchow/asset.yaml b/assets/models/system/Virchow/asset.yaml new file mode 100644 index 0000000000..fcf5c5a05b --- /dev/null +++ b/assets/models/system/Virchow/asset.yaml @@ -0,0 +1,4 @@ +extra_config: model.yaml +spec: spec.yaml +type: model +categories: ["Foundation Models"] diff --git a/assets/models/system/Virchow/description.md b/assets/models/system/Virchow/description.md new file mode 100644 index 0000000000..22dd34a2cf --- /dev/null +++ b/assets/models/system/Virchow/description.md @@ -0,0 +1,102 @@ +Virchow is a self-supervised vision transformer pretrained using 1.5M whole slide histopathology images. The model can be used as a tile-level feature extractor (frozen or finetuned) to achieve state-of-the-art results for a wide variety of downstream computational pathology use cases. + +## Model Details +- **Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA +- **Model Type:** Image feature backbone +- **Model Stats:** + - Params (M): 632 + - Image size: 224 x 224 +- **Model Architecture:** + - Architecture: ViT-H/14 + - Patch size: 14 + - Layers: 32 + - Embedding dimension: 1280 + - Activation function: SwiGLU + - Attention heads: 16 + - LayerScale: true +- **Training Details:**: + - Precision: Mixed precision (`fp16`) + - Objective: DINOv2 (https://doi.org/10.48550/arXiv.2304.07193) +- **Paper:** + - A foundation model for clinical-grade computational pathology and rare cancers detection: https://www.nature.com/articles/s41591-024-03141-0 +- **Pretraining Dataset:** Internal dataset of 1.5 million whole slide images from Memorial Sloan Kettering Cancer Center, tiles sampled at 0.5 microns per pixel resolution (20x magnification). +- **License:** Apache 2.0 + +## Model Usage + +### Sample Input + +```json +data = { + "input_data": { + "columns": [ + "image" + ], + "index":[0], + "data": [ + ["image1"] + ] + } +} +``` +Note: +- "image1" and "image2" should be publicly accessible urls or strings in base64 format. + +### Sample Output +```json +[ + { + "output": [ + 0.0, 0.0, 0.0, 0.0 + ] + } +] +``` +Output will be image embeddings. + +## Use +### Direct use +Virchow intended to be used as a frozen feature extractor as the foundation for tile-level and whole slide-level classifiers. +### Downstream use +Virchow can be finetuned to adapt to specific tasks and/or datasets. +## Terms of use +The Virchow Model and associated code are released under the Apache License, Version 2.0 (the "License"). You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Additional Terms +Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow Model requires prior registration on Hugging Face and agreeing to the terms of use. + +While the Apache 2.0 License grants broad permissions, we kindly request that users adhere to the following guidelines: + + 1. Attribution: We encourage proper attribution when using or redistributing the Virchow Model or its derivatives. Please include a reference to the original source and creators. + + 2. Responsible Use: Users are expected to use the Virchow Model responsibly and ethically. Please consider the potential impacts of your use on individuals and society. + + 3. Medical or Clinical Use: The Virchow Model is not intended for use in medical diagnosis, treatment, or prevention of disease of real patients. It should not be used as a substitute for professional medical advice. + + 4. Privacy and Data Protection: Users should respect privacy rights and comply with applicable data protection laws when using the Virchow Model. + + 5. No Malicious Use: The Virchow Model should not be used to create malicious code, malware, or to interfere with the proper functioning of computer systems. + + 6. Transparency: If you use the Virchow Model in a product or service, we encourage you to disclose this fact to your end users. + + 7. Feedback and Contributions: We welcome feedback and contributions to improve the Virchow Model. Please consider sharing your improvements with the community. + +These additional terms are not intended to restrict your rights under the Apache 2.0 License but to promote responsible and ethical use of the Virchow Model. + +By using the Virchow Model, you acknowledge that you have read and understood these terms. +## Citation +Please cite the following work if you used this model in your research. +Vorontsov, E., Bozkurt, A., Casson, A. et al. A foundation model for clinical-grade computational pathology and rare cancers detection. Nat Med (2024). https://doi.org/10.1038/s41591-024-03141-0 +``` +@article{vorontsov2024virchow, + title={A foundation model for clinical-grade computational pathology and rare cancers detection}, + author={Vorontsov, Eugene and Bozkurt, Alican and Casson, Adam and Shaikovski, George and Zelechowski, Michal and Severson, Kristen and Zimmermann, Eric and Hall, James and Tenenholtz, Neil and Fusi, Nicolo and Yang, Ellen and Mathieu, Philippe and van Eck, Alexander and Lee, Donghun and Viret, Julian and Robert, Eric and Wang, Yi Kan and Kunz, Jeremy D. and Lee, Matthew C. H. and Bernhard, Jan H. and Godrich, Ran A. and Oakley, Gerard and Millar, Ewan and Hanna, Matthew and Wen, Hannah and Retamero, Juan A. and Moye, William A. and Yousfi, Razik and Kanan, Christopher and Klimstra, David S. and Rothrock, Brandon and Liu, Siqi and Fuchs, Thomas J.}, + journal={Nature Medicine}, + year={2024}, + publisher={Nature Publishing Group} +} +``` \ No newline at end of file diff --git a/assets/models/system/Virchow/model.yaml b/assets/models/system/Virchow/model.yaml new file mode 100644 index 0000000000..41d6c6d666 --- /dev/null +++ b/assets/models/system/Virchow/model.yaml @@ -0,0 +1,8 @@ +path: + container_name: models + container_path: huggingface/Virchow/mlflow_model_folder + storage_name: automlcesdkdataresources + type: azureblob +publish: + description: description.md + type: mlflow_model diff --git a/assets/models/system/Virchow/spec.yaml b/assets/models/system/Virchow/spec.yaml new file mode 100644 index 0000000000..862201e022 --- /dev/null +++ b/assets/models/system/Virchow/spec.yaml @@ -0,0 +1,34 @@ +$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json + +name: Virchow +path: ./ + +properties: + inference-min-sku-spec: 6|1|112|64 + inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 + languages: en + SharedComputeCapacityEnabled: true + +tags: + task: image-feature-extraction + industry: health-and-life-sciences + Preview: "" + inference_supported_envs: + - hf + license: apache-2.0 + author: Paige + hiddenlayerscanned: "" + SharedComputeCapacityEnabled: "" + inference_compute_allow_list: + [ + Standard_NC6s_v3, + Standard_NC12s_v3, + Standard_NC24s_v3, + Standard_NC24ads_A100_v4, + Standard_NC48ads_A100_v4, + Standard_NC96ads_A100_v4, + Standard_ND96asr_v4, + Standard_ND96amsr_A100_v4, + Standard_ND40rs_v2, + ] +version: 1 \ No newline at end of file diff --git a/assets/models/system/Virchow2/asset.yaml b/assets/models/system/Virchow2/asset.yaml new file mode 100644 index 0000000000..fcf5c5a05b --- /dev/null +++ b/assets/models/system/Virchow2/asset.yaml @@ -0,0 +1,4 @@ +extra_config: model.yaml +spec: spec.yaml +type: model +categories: ["Foundation Models"] diff --git a/assets/models/system/Virchow2/description.md b/assets/models/system/Virchow2/description.md new file mode 100644 index 0000000000..e1c592e47b --- /dev/null +++ b/assets/models/system/Virchow2/description.md @@ -0,0 +1,105 @@ +Virchow2 is a self-supervised vision transformer pretrained using 3.1M whole slide histopathology images. The model can be used as a tile-level feature extractor (frozen or finetuned) to achieve state-of-the-art results for a wide variety of downstream computational pathology use cases. + +## Model Details +- **Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA +- **Model Type:** Image feature backbone +- **Model Stats:** + - Params (M): 632 + - Image size: 224 x 224 +- **Model Architecture:** + - Architecture: ViT-H/14 + - Patch size: 14 + - Layers: 32 + - Embedding dimension: 1280 + - Activation function: SwiGLU + - Attention heads: 16 + - LayerScale: true + - Register tokens: 4 +- **Training Details:** + - Precision: Mixed precision (`fp16`) + - Objective: Modified DINOv2 (https://doi.org/10.48550/arXiv.2304.07193) + - KoLeo regularizer replaced with kernel density estimator + - Crop-and-resize augmentation replaced with extended context translation +- **Paper:** + - Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology https://arxiv.org/pdf/2408.00738 +- **Pretraining Dataset:** Internal dataset of 3.1 million whole slide images from Memorial Sloan Kettering Cancer Center, tiles sampled at 2.0, 1.0, 0.5 and 0.25 microns per pixel resolution (5x, 10x, 20x, and 40x magnification). +- **License:** CC-BY-NC-ND-4.0 + +## Model Usage + +### Sample Input + +```json +data = { + "input_data": { + "columns": [ + "image" + ], + "index":[0], + "data": [ + ["image1"] + ] + } +} +``` +Note: +- "image1" and "image2" should be publicly accessible urls or strings in base64 format. + +### Sample Output +```json +[ + { + "output": [ + 0.0, 0.0, 0.0, 0.0 + ] + } +] +``` +Output will be image embeddings. + +## Use +### Direct use +Virchow2 intended to be used as a frozen feature extractor as the foundation for tile-level and whole slide-level classifiers. +### Downstream use +Virchow2 can be finetuned to adapt to specific tasks and/or datasets. +## Terms of use +This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. +Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: + + 1. Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. + + 2. Re-identify the deidentified data used to develop the Virchow2 Model; + + 3. Violate the law or others’ rights, including to: + a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; + + b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; + + c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; + + d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; + + e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; + + f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Virchow2 Model or any related materials; and + + g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. + + 5. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the Virchow2 Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and + + 6. Intentionally deceive or mislead others, including representing that the use of the Virchow2 Model or its outputs is human-generated. +Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. +## Citation +Please cite the following work if you used this model in your research. +Zimmermann, E., Vorontsov, E., Viret, J. et al. Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology. arXiv preprint arXiv:2408.00738 (2024). +``` +@article{zimmermann2024virchow2, + title={Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology}, + author={Eric Zimmermann and Eugene Vorontsov and Julian Viret and Adam Casson and Michal Zelechowski and George Shaikovski and Neil Tenenholtz and James Hall and Thomas Fuchs and Nicolo Fusi and Siqi Liu and Kristen Severson}, + journal={arXiv preprint arXiv:2408.00738}, + year={2024}, +} +``` +## Disclaimer +Virchow2 has been developed for research purposes and is not intended for diagnosis of real patients or projection/prediction of future disease possibilities. +Fairness evaluation cannot be completed due to limitations in the metadata. Underlying biases of the training datasets may not be well characterized and may not be representative of all demographics. \ No newline at end of file diff --git a/assets/models/system/Virchow2/model.yaml b/assets/models/system/Virchow2/model.yaml new file mode 100644 index 0000000000..56b3e57960 --- /dev/null +++ b/assets/models/system/Virchow2/model.yaml @@ -0,0 +1,8 @@ +path: + container_name: models + container_path: huggingface/Virchow2/mlflow_model_folder + storage_name: automlcesdkdataresources + type: azureblob +publish: + description: description.md + type: mlflow_model diff --git a/assets/models/system/Virchow2/spec.yaml b/assets/models/system/Virchow2/spec.yaml new file mode 100644 index 0000000000..f5bdd5a973 --- /dev/null +++ b/assets/models/system/Virchow2/spec.yaml @@ -0,0 +1,34 @@ +$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json + +name: Virchow2 +path: ./ + +properties: + inference-min-sku-spec: 6|1|112|64 + inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 + languages: en + SharedComputeCapacityEnabled: true + +tags: + task: image-feature-extraction + industry: health-and-life-sciences + Preview: "" + inference_supported_envs: + - hf + license: cc-by-nc-nd-4.0 + author: Paige + hiddenlayerscanned: "" + SharedComputeCapacityEnabled: "" + inference_compute_allow_list: + [ + Standard_NC6s_v3, + Standard_NC12s_v3, + Standard_NC24s_v3, + Standard_NC24ads_A100_v4, + Standard_NC48ads_A100_v4, + Standard_NC96ads_A100_v4, + Standard_ND96asr_v4, + Standard_ND96amsr_A100_v4, + Standard_ND40rs_v2, + ] +version: 1 \ No newline at end of file From 63bcef15eb61d22169ca1338affaecd8d7a67454 Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Wed, 9 Oct 2024 19:07:16 -0700 Subject: [PATCH 2/8] fix json --- .../description.md | 2 +- assets/models/system/Virchow/description.md | 2 +- assets/models/system/Virchow2/description.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md index 24d31af43d..dc5e56b22c 100644 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md @@ -21,7 +21,7 @@ It can perform various vision-language processing (VLP) tasks such as cross-moda ### Sample Input ```json -data = { +{ "input_data": { "columns": [ "image", diff --git a/assets/models/system/Virchow/description.md b/assets/models/system/Virchow/description.md index 22dd34a2cf..166dae1881 100644 --- a/assets/models/system/Virchow/description.md +++ b/assets/models/system/Virchow/description.md @@ -27,7 +27,7 @@ Virchow is a self-supervised vision transformer pretrained using 1.5M whole slid ### Sample Input ```json -data = { +{ "input_data": { "columns": [ "image" diff --git a/assets/models/system/Virchow2/description.md b/assets/models/system/Virchow2/description.md index e1c592e47b..43437885a4 100644 --- a/assets/models/system/Virchow2/description.md +++ b/assets/models/system/Virchow2/description.md @@ -30,7 +30,7 @@ Virchow2 is a self-supervised vision transformer pretrained using 3.1M whole sli ### Sample Input ```json -data = { +{ "input_data": { "columns": [ "image" From c848d296d08422e06d2e29f5af05a059b14f41f8 Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Wed, 9 Oct 2024 20:50:52 -0700 Subject: [PATCH 3/8] remove already added models --- .../asset.yaml | 4 - .../description.md | 75 --------- .../model.yaml | 8 - .../spec.yaml | 34 ---- assets/models/system/Virchow/asset.yaml | 4 - assets/models/system/Virchow/description.md | 102 ------------ assets/models/system/Virchow/model.yaml | 8 - assets/models/system/Virchow/spec.yaml | 34 ---- assets/models/system/Virchow2/description.md | 153 ++++++++++-------- 9 files changed, 82 insertions(+), 340 deletions(-) delete mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml delete mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md delete mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml delete mode 100644 assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml delete mode 100644 assets/models/system/Virchow/asset.yaml delete mode 100644 assets/models/system/Virchow/description.md delete mode 100644 assets/models/system/Virchow/model.yaml delete mode 100644 assets/models/system/Virchow/spec.yaml diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml deleted file mode 100644 index fcf5c5a05b..0000000000 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/asset.yaml +++ /dev/null @@ -1,4 +0,0 @@ -extra_config: model.yaml -spec: spec.yaml -type: model -categories: ["Foundation Models"] diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md deleted file mode 100644 index dc5e56b22c..0000000000 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md +++ /dev/null @@ -1,75 +0,0 @@ - -[BiomedCLIP](https://aka.ms/biomedclip-paper) is a biomedical vision-language foundation model that is pretrained on [PMC-15M](https://aka.ms/biomedclip-paper), a dataset of 15 million figure-caption pairs extracted from biomedical research articles in PubMed Central, using contrastive learning. -It uses PubMedBERT as the text encoder and Vision Transformer as the image encoder, with domain-specific adaptations. -It can perform various vision-language processing (VLP) tasks such as cross-modal retrieval, image classification, and visual question answering. - -## Citation - -```bibtex -@misc{https://doi.org/10.48550/arXiv.2303.00915, - doi = {10.48550/ARXIV.2303.00915}, - url = {https://arxiv.org/abs/2303.00915}, - author = {Zhang, Sheng and Xu, Yanbo and Usuyama, Naoto and Bagga, Jaspreet and Tinn, Robert and Preston, Sam and Rao, Rajesh and Wei, Mu and Valluri, Naveen and Wong, Cliff and Lungren, Matthew and Naumann, Tristan and Poon, Hoifung}, - title = {Large-Scale Domain-Specific Pretraining for Biomedical Vision-Language Processing}, - publisher = {arXiv}, - year = {2023}, -} -``` - -## Model Use - -### Sample Input - -```json -{ - "input_data": { - "columns": [ - "image", - "text" - ], - "index":[0, 1, 2], - "data": [ - ["image1", "labe1, label2, label3"], - ["image2", "labe1, label2, label3"], - ["image3", "labe1, label2, label3"], - ] - } -} -``` -### Sample Output -```json -[ - { - "probs": [0.95, 0.03, 0.02], - "labels": ["label1", "label2", "label3"] - }, - { - "probs": [0.04, 0.93, 0.03], - "labels": ["label1", "label2", "label3"] - } -] -``` - -### Intended Use - -This model is intended to be used solely for (I) future research on visual-language processing and (II) reproducibility of the experimental results reported in the reference paper. - -#### Primary Intended Use - -The primary intended use is to support AI researchers building on top of this work. BiomedCLIP and its associated models should be helpful for exploring various biomedical VLP research questions, especially in the radiology domain. - -#### Out-of-Scope Use - -**Any** deployed use case of the model --- commercial or otherwise --- is currently out of scope. Although we evaluated the models using a broad set of publicly-available research benchmarks, the models and evaluations are not intended for deployed use cases. Please refer to [the associated paper](https://aka.ms/biomedclip-paper) for more details. - -## Data - -This model builds upon [PMC-15M dataset](https://aka.ms/biomedclip-paper), which is a large-scale parallel image-text dataset for biomedical vision-language processing. It contains 15 million figure-caption pairs extracted from biomedical research articles in PubMed Central. It covers a diverse range of biomedical image types, such as microscopy, radiography, histology, and more. - -## Limitations - -This model was developed using English corpora, and thus can be considered English-only. - -## Further information - -Please refer to the corresponding paper, ["Large-Scale Domain-Specific Pretraining for Biomedical Vision-Language Processing"](https://aka.ms/biomedclip-paper) for additional details on the model training and evaluation. \ No newline at end of file diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml deleted file mode 100644 index 01fd42f578..0000000000 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/model.yaml +++ /dev/null @@ -1,8 +0,0 @@ -path: - container_name: models - container_path: huggingface/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/mlflow_model_folder - storage_name: automlcesdkdataresources - type: azureblob -publish: - description: description.md - type: mlflow_model diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml deleted file mode 100644 index 0d0ce199b4..0000000000 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/spec.yaml +++ /dev/null @@ -1,34 +0,0 @@ -$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json - -name: BiomedCLIP-PubMedBERT_256-vit_base_patch16_224 -path: ./ - -properties: - inference-min-sku-spec: 6|1|112|64 - inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 - languages: en - SharedComputeCapacityEnabled: true - -tags: - task: zero-shot-image-classification - industry: health-and-life-sciences - Preview: "" - inference_supported_envs: - - hf - license: mit - author: Microsoft - hiddenlayerscanned: "" - SharedComputeCapacityEnabled: "" - inference_compute_allow_list: - [ - Standard_NC6s_v3, - Standard_NC12s_v3, - Standard_NC24s_v3, - Standard_NC24ads_A100_v4, - Standard_NC48ads_A100_v4, - Standard_NC96ads_A100_v4, - Standard_ND96asr_v4, - Standard_ND96amsr_A100_v4, - Standard_ND40rs_v2, - ] -version: 1 \ No newline at end of file diff --git a/assets/models/system/Virchow/asset.yaml b/assets/models/system/Virchow/asset.yaml deleted file mode 100644 index fcf5c5a05b..0000000000 --- a/assets/models/system/Virchow/asset.yaml +++ /dev/null @@ -1,4 +0,0 @@ -extra_config: model.yaml -spec: spec.yaml -type: model -categories: ["Foundation Models"] diff --git a/assets/models/system/Virchow/description.md b/assets/models/system/Virchow/description.md deleted file mode 100644 index 166dae1881..0000000000 --- a/assets/models/system/Virchow/description.md +++ /dev/null @@ -1,102 +0,0 @@ -Virchow is a self-supervised vision transformer pretrained using 1.5M whole slide histopathology images. The model can be used as a tile-level feature extractor (frozen or finetuned) to achieve state-of-the-art results for a wide variety of downstream computational pathology use cases. - -## Model Details -- **Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA -- **Model Type:** Image feature backbone -- **Model Stats:** - - Params (M): 632 - - Image size: 224 x 224 -- **Model Architecture:** - - Architecture: ViT-H/14 - - Patch size: 14 - - Layers: 32 - - Embedding dimension: 1280 - - Activation function: SwiGLU - - Attention heads: 16 - - LayerScale: true -- **Training Details:**: - - Precision: Mixed precision (`fp16`) - - Objective: DINOv2 (https://doi.org/10.48550/arXiv.2304.07193) -- **Paper:** - - A foundation model for clinical-grade computational pathology and rare cancers detection: https://www.nature.com/articles/s41591-024-03141-0 -- **Pretraining Dataset:** Internal dataset of 1.5 million whole slide images from Memorial Sloan Kettering Cancer Center, tiles sampled at 0.5 microns per pixel resolution (20x magnification). -- **License:** Apache 2.0 - -## Model Usage - -### Sample Input - -```json -{ - "input_data": { - "columns": [ - "image" - ], - "index":[0], - "data": [ - ["image1"] - ] - } -} -``` -Note: -- "image1" and "image2" should be publicly accessible urls or strings in base64 format. - -### Sample Output -```json -[ - { - "output": [ - 0.0, 0.0, 0.0, 0.0 - ] - } -] -``` -Output will be image embeddings. - -## Use -### Direct use -Virchow intended to be used as a frozen feature extractor as the foundation for tile-level and whole slide-level classifiers. -### Downstream use -Virchow can be finetuned to adapt to specific tasks and/or datasets. -## Terms of use -The Virchow Model and associated code are released under the Apache License, Version 2.0 (the "License"). You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -Additional Terms -Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow Model requires prior registration on Hugging Face and agreeing to the terms of use. - -While the Apache 2.0 License grants broad permissions, we kindly request that users adhere to the following guidelines: - - 1. Attribution: We encourage proper attribution when using or redistributing the Virchow Model or its derivatives. Please include a reference to the original source and creators. - - 2. Responsible Use: Users are expected to use the Virchow Model responsibly and ethically. Please consider the potential impacts of your use on individuals and society. - - 3. Medical or Clinical Use: The Virchow Model is not intended for use in medical diagnosis, treatment, or prevention of disease of real patients. It should not be used as a substitute for professional medical advice. - - 4. Privacy and Data Protection: Users should respect privacy rights and comply with applicable data protection laws when using the Virchow Model. - - 5. No Malicious Use: The Virchow Model should not be used to create malicious code, malware, or to interfere with the proper functioning of computer systems. - - 6. Transparency: If you use the Virchow Model in a product or service, we encourage you to disclose this fact to your end users. - - 7. Feedback and Contributions: We welcome feedback and contributions to improve the Virchow Model. Please consider sharing your improvements with the community. - -These additional terms are not intended to restrict your rights under the Apache 2.0 License but to promote responsible and ethical use of the Virchow Model. - -By using the Virchow Model, you acknowledge that you have read and understood these terms. -## Citation -Please cite the following work if you used this model in your research. -Vorontsov, E., Bozkurt, A., Casson, A. et al. A foundation model for clinical-grade computational pathology and rare cancers detection. Nat Med (2024). https://doi.org/10.1038/s41591-024-03141-0 -``` -@article{vorontsov2024virchow, - title={A foundation model for clinical-grade computational pathology and rare cancers detection}, - author={Vorontsov, Eugene and Bozkurt, Alican and Casson, Adam and Shaikovski, George and Zelechowski, Michal and Severson, Kristen and Zimmermann, Eric and Hall, James and Tenenholtz, Neil and Fusi, Nicolo and Yang, Ellen and Mathieu, Philippe and van Eck, Alexander and Lee, Donghun and Viret, Julian and Robert, Eric and Wang, Yi Kan and Kunz, Jeremy D. and Lee, Matthew C. H. and Bernhard, Jan H. and Godrich, Ran A. and Oakley, Gerard and Millar, Ewan and Hanna, Matthew and Wen, Hannah and Retamero, Juan A. and Moye, William A. and Yousfi, Razik and Kanan, Christopher and Klimstra, David S. and Rothrock, Brandon and Liu, Siqi and Fuchs, Thomas J.}, - journal={Nature Medicine}, - year={2024}, - publisher={Nature Publishing Group} -} -``` \ No newline at end of file diff --git a/assets/models/system/Virchow/model.yaml b/assets/models/system/Virchow/model.yaml deleted file mode 100644 index 41d6c6d666..0000000000 --- a/assets/models/system/Virchow/model.yaml +++ /dev/null @@ -1,8 +0,0 @@ -path: - container_name: models - container_path: huggingface/Virchow/mlflow_model_folder - storage_name: automlcesdkdataresources - type: azureblob -publish: - description: description.md - type: mlflow_model diff --git a/assets/models/system/Virchow/spec.yaml b/assets/models/system/Virchow/spec.yaml deleted file mode 100644 index 862201e022..0000000000 --- a/assets/models/system/Virchow/spec.yaml +++ /dev/null @@ -1,34 +0,0 @@ -$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json - -name: Virchow -path: ./ - -properties: - inference-min-sku-spec: 6|1|112|64 - inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 - languages: en - SharedComputeCapacityEnabled: true - -tags: - task: image-feature-extraction - industry: health-and-life-sciences - Preview: "" - inference_supported_envs: - - hf - license: apache-2.0 - author: Paige - hiddenlayerscanned: "" - SharedComputeCapacityEnabled: "" - inference_compute_allow_list: - [ - Standard_NC6s_v3, - Standard_NC12s_v3, - Standard_NC24s_v3, - Standard_NC24ads_A100_v4, - Standard_NC48ads_A100_v4, - Standard_NC96ads_A100_v4, - Standard_ND96asr_v4, - Standard_ND96amsr_A100_v4, - Standard_ND40rs_v2, - ] -version: 1 \ No newline at end of file diff --git a/assets/models/system/Virchow2/description.md b/assets/models/system/Virchow2/description.md index 43437885a4..381ff3d215 100644 --- a/assets/models/system/Virchow2/description.md +++ b/assets/models/system/Virchow2/description.md @@ -1,32 +1,90 @@ Virchow2 is a self-supervised vision transformer pretrained using 3.1M whole slide histopathology images. The model can be used as a tile-level feature extractor (frozen or finetuned) to achieve state-of-the-art results for a wide variety of downstream computational pathology use cases. + ## Model Details -- **Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA -- **Model Type:** Image feature backbone -- **Model Stats:** - - Params (M): 632 - - Image size: 224 x 224 -- **Model Architecture:** - - Architecture: ViT-H/14 - - Patch size: 14 - - Layers: 32 - - Embedding dimension: 1280 - - Activation function: SwiGLU - - Attention heads: 16 - - LayerScale: true - - Register tokens: 4 -- **Training Details:** - - Precision: Mixed precision (`fp16`) - - Objective: Modified DINOv2 (https://doi.org/10.48550/arXiv.2304.07193) - - KoLeo regularizer replaced with kernel density estimator - - Crop-and-resize augmentation replaced with extended context translation -- **Paper:** - - Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology https://arxiv.org/pdf/2408.00738 -- **Pretraining Dataset:** Internal dataset of 3.1 million whole slide images from Memorial Sloan Kettering Cancer Center, tiles sampled at 2.0, 1.0, 0.5 and 0.25 microns per pixel resolution (5x, 10x, 20x, and 40x magnification). -- **License:** CC-BY-NC-ND-4.0 + +**Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA +**Model Type:** Image feature backbone +**Model Stats:** + Params (M): 632 + Image size: 224 x 224 + Model Architecture: +**Architecture:** ViT-H/14 + Patch size: 14 + Layers: 32 + Embedding dimension: 1280 + Activation function: SwiGLU + Attention heads: 16 + LayerScale: true + Register tokens: 4 +**Training Details:** + Precision: Mixed precision (fp16) + Objective: Modified DINOv2 (https://doi.org/10.48550/arXiv.2304.07193) + KoLeo regularizer replaced with kernel density estimator + Crop-and-resize augmentation replaced with extended context translation +**Paper:** + Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology https://arxiv.org/pdf/2408.00738 +**Pretraining Dataset:** Internal dataset of 3.1 million whole slide images from Memorial Sloan Kettering Cancer Center, tiles sampled at 2.0, 1.0, 0.5 and 0.25 microns per pixel resolution (5x, 10x, 20x, and 40x magnification). +**License:** CC-BY-NC-ND-4.0 ## Model Usage +**Direct use** +Virchow2 intended to be used as a frozen feature extractor as the foundation for tile-level and whole slide-level classifiers. + +**Downstream use** +Virchow2 can be finetuned to adapt to specific tasks and/or datasets. + +**Terms** + +This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: + +Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. + +Re-identify the deidentified data used to develop the Virchow2 Model; + +Violate the law or others’ rights, including to: + +a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; + +b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; + +c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; + +d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; + +e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; + +f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Virchow2 Model or any related materials; and + +g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. + +Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the Virchow2 Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and + +Intentionally deceive or mislead others, including representing that the use of the Virchow2 Model or its outputs is human-generated. + +Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. + +**Citation** + +Please cite the following work if you used this model in your research. + +Zimmermann, E., Vorontsov, E., Viret, J. et al. Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology. arXiv preprint arXiv:2408.00738 (2024). + +``` +@article{zimmermann2024virchow2, + title={Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology}, + author={Eric Zimmermann and Eugene Vorontsov and Julian Viret and Adam Casson and Michal Zelechowski and George Shaikovski and Neil Tenenholtz and James Hall and Thomas Fuchs and Nicolo Fusi and Siqi Liu and Kristen Severson}, + journal={arXiv preprint arXiv:2408.00738}, + year={2024}, +} +``` +**Disclaimer** +Virchow2 has been developed for research purposes and is not intended for diagnosis of real patients or projection/prediction of future disease possibilities. + +Fairness evaluation cannot be completed due to limitations in the metadata. Underlying biases of the training datasets may not be well characterized and may not be representative of all demographics. + +## Sample Input and Output (for real-time inference) ### Sample Input ```json @@ -55,51 +113,4 @@ Note: } ] ``` -Output will be image embeddings. - -## Use -### Direct use -Virchow2 intended to be used as a frozen feature extractor as the foundation for tile-level and whole slide-level classifiers. -### Downstream use -Virchow2 can be finetuned to adapt to specific tasks and/or datasets. -## Terms of use -This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. -Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: - - 1. Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. - - 2. Re-identify the deidentified data used to develop the Virchow2 Model; - - 3. Violate the law or others’ rights, including to: - a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; - - b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; - - c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; - - d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; - - e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; - - f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Virchow2 Model or any related materials; and - - g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. - - 5. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the Virchow2 Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and - - 6. Intentionally deceive or mislead others, including representing that the use of the Virchow2 Model or its outputs is human-generated. -Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. -## Citation -Please cite the following work if you used this model in your research. -Zimmermann, E., Vorontsov, E., Viret, J. et al. Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology. arXiv preprint arXiv:2408.00738 (2024). -``` -@article{zimmermann2024virchow2, - title={Virchow2: Scaling Self-Supervised Mixed Magnification Models in Pathology}, - author={Eric Zimmermann and Eugene Vorontsov and Julian Viret and Adam Casson and Michal Zelechowski and George Shaikovski and Neil Tenenholtz and James Hall and Thomas Fuchs and Nicolo Fusi and Siqi Liu and Kristen Severson}, - journal={arXiv preprint arXiv:2408.00738}, - year={2024}, -} -``` -## Disclaimer -Virchow2 has been developed for research purposes and is not intended for diagnosis of real patients or projection/prediction of future disease possibilities. -Fairness evaluation cannot be completed due to limitations in the metadata. Underlying biases of the training datasets may not be well characterized and may not be representative of all demographics. \ No newline at end of file +Output will be image embeddings. \ No newline at end of file From d54781b59cb4c1cba2c8fa2035758f6f76231f9d Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Wed, 9 Oct 2024 21:10:13 -0700 Subject: [PATCH 4/8] add prism assets --- assets/models/system/Prism/asset.yaml | 4 + assets/models/system/Prism/description.md | 98 +++++++++++++++++++++++ assets/models/system/Prism/model.yaml | 8 ++ assets/models/system/Prism/spec.yaml | 34 ++++++++ 4 files changed, 144 insertions(+) create mode 100644 assets/models/system/Prism/asset.yaml create mode 100644 assets/models/system/Prism/description.md create mode 100644 assets/models/system/Prism/model.yaml create mode 100644 assets/models/system/Prism/spec.yaml diff --git a/assets/models/system/Prism/asset.yaml b/assets/models/system/Prism/asset.yaml new file mode 100644 index 0000000000..fcf5c5a05b --- /dev/null +++ b/assets/models/system/Prism/asset.yaml @@ -0,0 +1,4 @@ +extra_config: model.yaml +spec: spec.yaml +type: model +categories: ["Foundation Models"] diff --git a/assets/models/system/Prism/description.md b/assets/models/system/Prism/description.md new file mode 100644 index 0000000000..16ec633c7f --- /dev/null +++ b/assets/models/system/Prism/description.md @@ -0,0 +1,98 @@ +PRISM is a multi-modal generative foundation model for slide-level analysis of H&E-stained histopathology images. Utilizing Virchow tile embeddings and clinical report texts for pre-training, PRISM combines these embeddings into a single slide embedding and generates a text-based diagnostic report. These can be used for tasks such as cancer detection, sub-typing, and biomarker identification. The model's slide encoder can be fine-tuned for specific classification tasks, leveraging both image and text data to enhance diagnostic performance and robustness. + +PRISM supports several modes of use: + +text report generation to describe tissue in H&E whole slide images +zero-shot cancer detection and sub-typing using text prompts +adaptation to new tasks via PRISM finetuning, or linear classifier on the slide embedding + +## Model Details + +**Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA +**Model Type:** Vision-Language Encoder-Decoder +**Model Stats:** + Params (M): 558 +**Architecture:** + Encoder: Perceiver (https://doi.org/10.48550/arXiv.2103.03206) + Decoder: BioGPT (https://huggingface.co/microsoft/biogpt) + Model inputs: tile image embeddings and text captions + Tile image encoder: Virchow V1 (https://huggingface.co/paige-ai/Virchow) +**Training Details:** + Objective: CoCa (https://doi.org/10.48550/arXiv.2205.01917) + Precision: Mixed precision (fp16) +**Paper:** + PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology: https://arxiv.org/abs/2405.10254 +**Pretraining Dataset:** Internal dataset of 587 thousand whole slide images and 195 thousand clinical reports from Memorial Sloan Kettering Cancer Center. +**License:** CC-BY-NC-ND-4.0 + +## Model Usage + +**Direct use** +PRISM is a vision-language model that can analyze whole slide images using the following methods: + +CLIP-style zero-shot classification via zero_shot method, or +generate a tissue description in the image via generate method. +The model takes whole slide images in the form of tile embeddings from our Virchow model. Please see https://huggingface.co/paige-ai/Virchow for instructions on how to use it to generate embeddings for your whole slide image. + +**Downstream use** +You can use PRISM to compute slide embedding for downstream tasks such as slide-level classification. The slide embedding can be further adapted to new tasks by finetuning the slide encoder of PRISM on slide-level labels, e.g. biomarkers. + +Slide embeddings are accessible via slide_representations method. + +**Terms of use** + +This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the PRISM Model and its derivatives, which include models trained on outputs from the PRISM Model or datasets created from the PRISM Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the PRISM Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the PRISM Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the PRISM model, you agree not to distribute, publish or reproduce a copy of the PRISM Model. If another user within your organization wishes to use the PRISM Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. + +Further, by downloading the PRISM model, you agree you will only use the PRISM model for academic research purposes and will not use, or allow others to use, the PRISM model to: + +Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. + +Re-identify the deidentified data used to develop the PRISM Model; + +Violate the law or others’ rights, including to: + +a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; + +b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; + +c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; + +d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; + +e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; + +f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the PRISM Model or any related materials; and + +g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. + +Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the PRISM Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and + +Intentionally deceive or mislead others, including representing that the use of the PRISM Model or its outputs is human-generated. + +Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. + +**Citation** + +Please cite the following work if you use this model in your research. + +Shaikovski, George, Adam Casson, Kristen Severson, Eric Zimmermann et al. "PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology." arXiv preprint arXiv:2405.10254 (2024). https://doi.org/10.48550/arXiv.2405.10254 + +``` +@article{shaikovski2024prism, + title={PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology}, + author={Shaikovski, George and Casson, Adam and Severson, Kristen and Zimmermann, Eric and Wang, Yi Kan and Kunz, Jeremy D and Retamero, Juan A and Oakley, Gerard and Klimstra, David and Kanan, Christopher and others}, + journal={arXiv preprint arXiv:2405.10254}, + year={2024} +} + +``` +**Disclaimer** + +PRISM has been developed for research purposes and is not intended for diagnosis of real patients or projection/prediction of future disease possibilities. + +Fairness evaluation cannot be completed due to limitations in the metadata. Underlying biases of the training datasets may not be well characterized and may not be representative of all demographics. + +**Acknowledgements** +The results shown here (specifically, in the section "Sample inference code") are in whole or part based upon data generated by the TCGA Research Network: http://cancergenome.nih.gov/. + +## Sample Input and Output (for real-time inference) \ No newline at end of file diff --git a/assets/models/system/Prism/model.yaml b/assets/models/system/Prism/model.yaml new file mode 100644 index 0000000000..65203f3732 --- /dev/null +++ b/assets/models/system/Prism/model.yaml @@ -0,0 +1,8 @@ +path: + container_name: models + container_path: huggingface/Prism/mlflow_model_folder + storage_name: automlcesdkdataresources + type: azureblob +publish: + description: description.md + type: mlflow_model diff --git a/assets/models/system/Prism/spec.yaml b/assets/models/system/Prism/spec.yaml new file mode 100644 index 0000000000..8bc3091785 --- /dev/null +++ b/assets/models/system/Prism/spec.yaml @@ -0,0 +1,34 @@ +$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json + +name: Prism +path: ./ + +properties: + inference-min-sku-spec: 6|1|112|64 + inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 + languages: en + SharedComputeCapacityEnabled: true + +tags: + task: zero-shot-image-classification + industry: health-and-life-sciences + Preview: "" + inference_supported_envs: + - hf + license: cc-by-nc-nd-4.0 + author: Paige + hiddenlayerscanned: "" + SharedComputeCapacityEnabled: "" + inference_compute_allow_list: + [ + Standard_NC6s_v3, + Standard_NC12s_v3, + Standard_NC24s_v3, + Standard_NC24ads_A100_v4, + Standard_NC48ads_A100_v4, + Standard_NC96ads_A100_v4, + Standard_ND96asr_v4, + Standard_ND96amsr_A100_v4, + Standard_ND40rs_v2, + ] +version: 1 \ No newline at end of file From 62bcc0f27fb4996cd6f64c77be6046dd7ae61f53 Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Thu, 10 Oct 2024 08:14:35 -0700 Subject: [PATCH 5/8] update model cards --- assets/models/system/Prism/description.md | 2 +- assets/models/system/Virchow2/description.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/models/system/Prism/description.md b/assets/models/system/Prism/description.md index 16ec633c7f..339521d32e 100644 --- a/assets/models/system/Prism/description.md +++ b/assets/models/system/Prism/description.md @@ -41,7 +41,7 @@ Slide embeddings are accessible via slide_representations method. **Terms of use** -This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the PRISM Model and its derivatives, which include models trained on outputs from the PRISM Model or datasets created from the PRISM Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the PRISM Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the PRISM Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the PRISM model, you agree not to distribute, publish or reproduce a copy of the PRISM Model. If another user within your organization wishes to use the PRISM Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. +This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the PRISM Model and its derivatives, which include models trained on outputs from the PRISM Model or datasets created from the PRISM Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the PRISM Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the PRISM Model requires prior registration on Azure AI Studio and agreeing to the terms of use. By downloading the PRISM model, you agree not to distribute, publish or reproduce a copy of the PRISM Model. If another user within your organization wishes to use the PRISM Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading the PRISM model, you agree you will only use the PRISM model for academic research purposes and will not use, or allow others to use, the PRISM model to: diff --git a/assets/models/system/Virchow2/description.md b/assets/models/system/Virchow2/description.md index 381ff3d215..dd2ac6dc0b 100644 --- a/assets/models/system/Virchow2/description.md +++ b/assets/models/system/Virchow2/description.md @@ -37,7 +37,7 @@ Virchow2 can be finetuned to adapt to specific tasks and/or datasets. **Terms** -This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Hugging Face and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: +This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Azure AI Studio and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. From 16b8efd0467ad567ded69d48aadb7b0e446bccfc Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Thu, 10 Oct 2024 08:16:45 -0700 Subject: [PATCH 6/8] update Virchow card --- assets/models/system/Virchow/description.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/assets/models/system/Virchow/description.md b/assets/models/system/Virchow/description.md index c8179cd652..9947571939 100644 --- a/assets/models/system/Virchow/description.md +++ b/assets/models/system/Virchow/description.md @@ -41,7 +41,7 @@ Unless required by applicable law or agreed to in writing, software distributed Additional Terms -Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow Model requires prior registration on Hugging Face and agreeing to the terms of use. +By downloading the Virchow Model, you attest that all account information (affiliation, research use) is correct and up-to-date. Downloading the Virchow Model requires prior registration on Azure AI Studio and agreeing to the terms of use. While the Apache 2.0 License grants broad permissions, we kindly request that users adhere to the following guidelines: @@ -81,7 +81,6 @@ Vorontsov, E., Bozkurt, A., Casson, A. et al. A foundation model for clinical-gr ``` ## Sample Input and Output (for real-time inference) - ### Sample Input ```json From 15427fe1d9276a351abb156680009d7869702474 Mon Sep 17 00:00:00 2001 From: cassieesvelt Date: Thu, 10 Oct 2024 14:24:13 -0700 Subject: [PATCH 7/8] remove prism + fix biomedclip image --- .../description.md | 2 +- assets/models/system/Prism/asset.yaml | 4 - assets/models/system/Prism/description.md | 98 ------------------- assets/models/system/Prism/model.yaml | 8 -- assets/models/system/Prism/spec.yaml | 34 ------- 5 files changed, 1 insertion(+), 145 deletions(-) delete mode 100644 assets/models/system/Prism/asset.yaml delete mode 100644 assets/models/system/Prism/description.md delete mode 100644 assets/models/system/Prism/model.yaml delete mode 100644 assets/models/system/Prism/spec.yaml diff --git a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md index 522272f304..c6d242d4f4 100644 --- a/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md +++ b/assets/models/system/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/description.md @@ -1,6 +1,6 @@ BiomedCLIP is a biomedical vision-language foundation model that is pretrained on PMC-15M, a dataset of 15 million figure-caption pairs extracted from biomedical research articles in PubMed Central, using contrastive learning. It uses PubMedBERT as the text encoder and Vision Transformer as the image encoder, with domain-specific adaptations. It can perform various vision-language processing (VLP) tasks such as cross-modal retrieval, image classification, and visual question answering. BiomedCLIP establishes new state of the art in a wide range of standard datasets, and substantially outperforms prior VLP approaches: -![performance](https://huggingface.co/microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/resolve/main/biomed-vlp-eval.svg) +Model Performance comparision chart **Citation** diff --git a/assets/models/system/Prism/asset.yaml b/assets/models/system/Prism/asset.yaml deleted file mode 100644 index fcf5c5a05b..0000000000 --- a/assets/models/system/Prism/asset.yaml +++ /dev/null @@ -1,4 +0,0 @@ -extra_config: model.yaml -spec: spec.yaml -type: model -categories: ["Foundation Models"] diff --git a/assets/models/system/Prism/description.md b/assets/models/system/Prism/description.md deleted file mode 100644 index 339521d32e..0000000000 --- a/assets/models/system/Prism/description.md +++ /dev/null @@ -1,98 +0,0 @@ -PRISM is a multi-modal generative foundation model for slide-level analysis of H&E-stained histopathology images. Utilizing Virchow tile embeddings and clinical report texts for pre-training, PRISM combines these embeddings into a single slide embedding and generates a text-based diagnostic report. These can be used for tasks such as cancer detection, sub-typing, and biomarker identification. The model's slide encoder can be fine-tuned for specific classification tasks, leveraging both image and text data to enhance diagnostic performance and robustness. - -PRISM supports several modes of use: - -text report generation to describe tissue in H&E whole slide images -zero-shot cancer detection and sub-typing using text prompts -adaptation to new tasks via PRISM finetuning, or linear classifier on the slide embedding - -## Model Details - -**Developed by:** Paige, NYC, USA and Microsoft Research, Cambridge, MA USA -**Model Type:** Vision-Language Encoder-Decoder -**Model Stats:** - Params (M): 558 -**Architecture:** - Encoder: Perceiver (https://doi.org/10.48550/arXiv.2103.03206) - Decoder: BioGPT (https://huggingface.co/microsoft/biogpt) - Model inputs: tile image embeddings and text captions - Tile image encoder: Virchow V1 (https://huggingface.co/paige-ai/Virchow) -**Training Details:** - Objective: CoCa (https://doi.org/10.48550/arXiv.2205.01917) - Precision: Mixed precision (fp16) -**Paper:** - PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology: https://arxiv.org/abs/2405.10254 -**Pretraining Dataset:** Internal dataset of 587 thousand whole slide images and 195 thousand clinical reports from Memorial Sloan Kettering Cancer Center. -**License:** CC-BY-NC-ND-4.0 - -## Model Usage - -**Direct use** -PRISM is a vision-language model that can analyze whole slide images using the following methods: - -CLIP-style zero-shot classification via zero_shot method, or -generate a tissue description in the image via generate method. -The model takes whole slide images in the form of tile embeddings from our Virchow model. Please see https://huggingface.co/paige-ai/Virchow for instructions on how to use it to generate embeddings for your whole slide image. - -**Downstream use** -You can use PRISM to compute slide embedding for downstream tasks such as slide-level classification. The slide embedding can be further adapted to new tasks by finetuning the slide encoder of PRISM on slide-level labels, e.g. biomarkers. - -Slide embeddings are accessible via slide_representations method. - -**Terms of use** - -This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the PRISM Model and its derivatives, which include models trained on outputs from the PRISM Model or datasets created from the PRISM Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the PRISM Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the PRISM Model requires prior registration on Azure AI Studio and agreeing to the terms of use. By downloading the PRISM model, you agree not to distribute, publish or reproduce a copy of the PRISM Model. If another user within your organization wishes to use the PRISM Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. - -Further, by downloading the PRISM model, you agree you will only use the PRISM model for academic research purposes and will not use, or allow others to use, the PRISM model to: - -Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. - -Re-identify the deidentified data used to develop the PRISM Model; - -Violate the law or others’ rights, including to: - -a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; - -b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; - -c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; - -d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; - -e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; - -f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the PRISM Model or any related materials; and - -g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. - -Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the PRISM Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and - -Intentionally deceive or mislead others, including representing that the use of the PRISM Model or its outputs is human-generated. - -Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. - -**Citation** - -Please cite the following work if you use this model in your research. - -Shaikovski, George, Adam Casson, Kristen Severson, Eric Zimmermann et al. "PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology." arXiv preprint arXiv:2405.10254 (2024). https://doi.org/10.48550/arXiv.2405.10254 - -``` -@article{shaikovski2024prism, - title={PRISM: A Multi-Modal Generative Foundation Model for Slide-Level Histopathology}, - author={Shaikovski, George and Casson, Adam and Severson, Kristen and Zimmermann, Eric and Wang, Yi Kan and Kunz, Jeremy D and Retamero, Juan A and Oakley, Gerard and Klimstra, David and Kanan, Christopher and others}, - journal={arXiv preprint arXiv:2405.10254}, - year={2024} -} - -``` -**Disclaimer** - -PRISM has been developed for research purposes and is not intended for diagnosis of real patients or projection/prediction of future disease possibilities. - -Fairness evaluation cannot be completed due to limitations in the metadata. Underlying biases of the training datasets may not be well characterized and may not be representative of all demographics. - -**Acknowledgements** -The results shown here (specifically, in the section "Sample inference code") are in whole or part based upon data generated by the TCGA Research Network: http://cancergenome.nih.gov/. - -## Sample Input and Output (for real-time inference) \ No newline at end of file diff --git a/assets/models/system/Prism/model.yaml b/assets/models/system/Prism/model.yaml deleted file mode 100644 index 65203f3732..0000000000 --- a/assets/models/system/Prism/model.yaml +++ /dev/null @@ -1,8 +0,0 @@ -path: - container_name: models - container_path: huggingface/Prism/mlflow_model_folder - storage_name: automlcesdkdataresources - type: azureblob -publish: - description: description.md - type: mlflow_model diff --git a/assets/models/system/Prism/spec.yaml b/assets/models/system/Prism/spec.yaml deleted file mode 100644 index 8bc3091785..0000000000 --- a/assets/models/system/Prism/spec.yaml +++ /dev/null @@ -1,34 +0,0 @@ -$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json - -name: Prism -path: ./ - -properties: - inference-min-sku-spec: 6|1|112|64 - inference-recommended-sku: Standard_NC6s_v3, Standard_NC12s_v3, Standard_NC24s_v3, Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4, Standard_ND40rs_v2 - languages: en - SharedComputeCapacityEnabled: true - -tags: - task: zero-shot-image-classification - industry: health-and-life-sciences - Preview: "" - inference_supported_envs: - - hf - license: cc-by-nc-nd-4.0 - author: Paige - hiddenlayerscanned: "" - SharedComputeCapacityEnabled: "" - inference_compute_allow_list: - [ - Standard_NC6s_v3, - Standard_NC12s_v3, - Standard_NC24s_v3, - Standard_NC24ads_A100_v4, - Standard_NC48ads_A100_v4, - Standard_NC96ads_A100_v4, - Standard_ND96asr_v4, - Standard_ND96amsr_A100_v4, - Standard_ND40rs_v2, - ] -version: 1 \ No newline at end of file From f68aae734db2f2278541d93ef830d359c306e88f Mon Sep 17 00:00:00 2001 From: Tina Manghnani Date: Thu, 10 Oct 2024 14:31:04 -0700 Subject: [PATCH 8/8] Update description.md --- assets/models/system/Virchow2/description.md | 47 ++++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/assets/models/system/Virchow2/description.md b/assets/models/system/Virchow2/description.md index dd2ac6dc0b..a130cd1b07 100644 --- a/assets/models/system/Virchow2/description.md +++ b/assets/models/system/Virchow2/description.md @@ -35,36 +35,25 @@ Virchow2 intended to be used as a frozen feature extractor as the foundation for **Downstream use** Virchow2 can be finetuned to adapt to specific tasks and/or datasets. -**Terms** - -This model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. Please note that the primary email used to sign up for your Hugging Face account must match your institutional email to receive approval. By downloading the Virchow2 Model, you attest that all information (affiliation, research use) is correct and up-to-date. Downloading the Virchow2 Model requires prior registration on Azure AI Studio and agreeing to the terms of use. By downloading the Virchow2 model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with the terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading the Virchow2 model, you agree you will only use the Virchow2 model for academic research purposes and will not use, or allow others to use, the Virchow2 model to: - -Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. - -Re-identify the deidentified data used to develop the Virchow2 Model; - -Violate the law or others’ rights, including to: - -a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; - -b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; - -c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; - -d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; - -e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; - -f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Virchow2 Model or any related materials; and - -g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. - -Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the Virchow2 Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and - -Intentionally deceive or mislead others, including representing that the use of the Virchow2 Model or its outputs is human-generated. - +**Terms of use** + +The Virchow2 Model and associated code are released under the CC-BY-NC-ND 4.0 license and may only be used for non-commercial, academic research purposes with proper attribution. Any commercial use, sale, or other monetization of the Virchow2 Model and its derivatives, which include models trained on outputs from the Virchow2 Model or datasets created from the Virchow2 Model, is prohibited and requires prior approval. By downloading /deploying the Virchow2 Model, you attest that all account information (affiliation, research use) is correct and up-to-date. By downloading/deploying the Virchow2 Model, you agree not to distribute, publish or reproduce a copy of the Virchow2 Model. If another user within your organization wishes to use the Virchow2 Model, they must register as an individual user and agree to comply with these terms of use. If you are a commercial entity, please contact the corresponding author. Further, by downloading/deploying the Virchow2 Model, you agree you will only use the Virchow2 Model for academic research purposes and will not use, or allow others to use, the Virchow2 Model to: + +1. Diagnose, cure, mitigate, treat, or prevent disease or any other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or other similar use, and including as a substitute for professional medical advice, a healthcare opinion, a diagnosis, treatment, or the clinical judgment of a healthcare professional, as no license or right is granted for any such purposes. +2. Re-identify the deidentified data used to develop the Virchow2 Model; +3. Violate the law or others’ rights, including to: + a. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content; + b. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals; + c. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services; + d. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices; + e. Collect, process, disclose, generate, or infer the identity of individuals or the health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws; + f. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Virchow2 Model or any related materials; and + g. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system. +4. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including the use of the Virchow2 Model as a medical device, clinical support, diagnostic tool, or other technology intended to be used in the diagnosis, cure, mitigation, treatment, or prevention of disease or other conditions, including for Investigational Use Only (“IUO”), Research Use Only (“RUO”), commercial, clinical or similar use; and +5. Intentionally deceive or mislead others, including representing that the use of the Virchow2 Model or its outputs is human-generated. Further, you agree that you will appropriately disclose to end users any known dangers of your AI system. + **Citation** Please cite the following work if you used this model in your research. @@ -113,4 +102,4 @@ Note: } ] ``` -Output will be image embeddings. \ No newline at end of file +Output will be image embeddings.