Merge branch 'main' into akoumparouli/use_is_initialized_from_mcore

.github/workflows/cicd-main.yml

@@ -62,11 +62,9 @@ jobs:
 #        --env HYDRA_FULL_ERROR=1
     steps:
     - name: Log into ACR (Azure Container Registry)  # this login is for the pushing step after
-      uses: azure/docker-login@v1
-      with:
-        login-server: nemoci.azurecr.io
-        password: ${{ secrets.ACR_PASSWORD }}
-        username: nemoci
+      run: |
+        # Login to Azure Container Registry
+        az acr login --name nemoci.azurecr.io
 
     - name: Checkout repository
       uses: actions/checkout@v2

tutorials/multimodal/DreamBooth Tutorial.ipynb

@@ -124,10 +124,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! python /opt/NeMo/examples/multimodal/foundation/clip/convert_external_clip_to_nemo.py \\\n",
+    "! python /opt/NeMo/examples/multimodal/vision_language_foundation/clip/convert_external_clip_to_nemo.py \\\n",
     "        --arch ViT-L-14 \\\n",
     "        --version openai \\\n",
-    "        --hparams_file /opt/NeMo/examples/multimodal/foundation/clip/conf/megatron_clip_VIT-L-14.yaml \\\n",
+    "        --hparams_file /opt/NeMo/examples/multimodal/vision_language_foundation/clip/conf/megatron_clip_VIT-L-14.yaml \\\n",
     "        --nemo_file /ckpts/openai.nemo"
    ]
   },
@@ -167,7 +167,7 @@
    "outputs": [],
    "source": [
     "## This is the example command for running dreambooth training\n",
-    "! python /opt/NeMo/examples/multimodal/generative/dreambooth/dreambooth.py \\\n",
+    "! python /opt/NeMo/examples/multimodal/text_to_image/dreambooth/dreambooth.py \\\n",
     "    model.unet_config.from_pretrained=/ckpts/unet.bin \\\n",
     "    model.unet_config.from_NeMo=False \\\n",
     "    model.first_stage_config.from_pretrained=/ckpts/vae.bin \\\n",
@@ -196,7 +196,7 @@
    "outputs": [],
    "source": [
     "## This is the example command for running DreamBooth inference\n",
-    "! torchrun /opt/NeMo/examples/multimodal/generative/dreambooth/dreambooth_infer.py \\\n",
+    "! torchrun /opt/NeMo/examples/multimodal/text_to_image/dreambooth/dreambooth_infer.py \\\n",
     "    model.restore_from_path='/opt/NeMo/tutorials/multimodal/nemo_experiments/Dreambooth/checkpoints/Dreambooth.nemo' \\\n",
     "    infer.num_images_per_prompt=4 \\\n",
     "    infer.inference_steps=50 \\\n",
@@ -270,4 +270,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

tutorials/multimodal/Multimodal Data Preparation.ipynb

@@ -660,4 +660,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

tutorials/multimodal/Stable Diffusion Tutorial.ipynb

@@ -9,7 +9,7 @@
     "# Stable Diffusion Training / Inference Tutorial\n",
     "\n",
     "### Note:\n",
-    "Currently, this notebook must be run in a NeMo container. An example command to launch the container:\n",
+    "Currently, this notebook must be run in a NeMo container (> 24.01). An example command to launch the container:\n",
     "\n",
     "```\n",
     "docker run --gpus all -it --rm -v <your_nemo_dir>:/opt/NeMo --shm-size=8g \\\n",
@@ -30,7 +30,7 @@
     "\n",
     "## Datasets\n",
     "\n",
-    "Please refer to [ADD LINK]() for how to prepare a training dataset for Stable diffusion.\n",
+    "Please refer to [Dataset Tutorial](https://github.com/NVIDIA/NeMo/blob/main/tutorials/multimodal/Multimodal%20Data%20Preparation.ipynb) for how to prepare a training dataset for Stable diffusion.\n",
     "\n",
     "For a pre-cached Stable Diffusion dataset, each webdataset tar file should, at a minimum, include the pickle files that store the pre-cached image and text features:\n",
     "\n",
@@ -117,10 +117,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! python examples/multimodal/foundation/clip/convert_external_clip_to_nemo.py \\\n",
+    "! python /opt/NeMo/examples/multimodal/vision_language_foundation/clip/convert_external_clip_to_nemo.py \\\n",
     "        --arch ViT-L-14 \\\n",
     "        --version openai \\\n",
-    "        --hparams_file /opt/NeMo/examples/multimodal/foundation/clip/conf/megatron_clip_VIT-L-14.yaml \\\n",
+    "        --hparams_file /opt/NeMo/examples/multimodal/vision_language_foundation/clip/conf/megatron_clip_VIT-L-14.yaml \\\n",
     "        --nemo_file /ckpts/openai.nemo"
    ]
   },
@@ -151,7 +151,7 @@
     "\n",
     "### Option 2: Training on Precached Dataset (Training UNet Only)\n",
     "\n",
-    "When using precached dataset (please refer to the [Dataset Tutorial](ADD_LINK) for details), every text feature and image feature are stored as key-value pairs in `.pickle` file:\n",
+    "When using precached dataset (please refer to the [Dataset Tutorial](https://github.com/NVIDIA/NeMo/blob/main/tutorials/multimodal/Multimodal%20Data%20Preparation.ipynb) for details), every text feature and image feature are stored as key-value pairs in `.pickle` file:\n",
     "\n",
     "```\n",
     "{\n",
@@ -201,7 +201,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! torchrun /opt/NeMo/examples/multimodal/generative/stable_diffusion/sd_train.py trainer.max_steps=100 model.data.synthetic_data=True"
+    "! torchrun /opt/NeMo/examples/multimodal/text_to_image/stable_diffusion/sd_train.py trainer.max_steps=100 model.data.synthetic_data=True"
    ]
   },
   {
@@ -247,7 +247,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! ! torchrun  /opt/NeMo/examples/multimodal/generative/stable_diffusion/sd_infer.py model.restore_from_path='/opt/NeMo/tutorials/multimodal/nemo_experiments/stable-diffusion-train/checkpoints/stable-diffusion-train.nemo'"
+    "! torchrun  /opt/NeMo/examples/multimodal/text_to_image/stable_diffusion/sd_infer.py model.restore_from_path='/opt/NeMo/tutorials/multimodal/nemo_experiments/stable-diffusion-train/checkpoints/stable-diffusion-train.nemo'"
    ]
   }
  ],
@@ -272,4 +272,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}