Merge branch 'master' into docs/rpi

Author: YunjinJo

.jenkins/get_files_to_run.py

@@ -1,4 +1,4 @@
-from typing import List, Any
+from typing import Any, Dict, List, Optional, Tuple
 import json
 import os
 from pathlib import Path
@@ -14,17 +14,21 @@ def get_all_files() -> List[str]:
     return [str(x) for x in sources]
 
 
-def calculate_shards(all_files: List[str], num_shards: int = 20):
+def read_metadata() -> Dict[str, Any]:
     with (REPO_BASE_DIR / ".jenkins" / "metadata.json").open() as fp:
-        metadata = json.load(fp)
-    sharded_files = [(0.0, []) for _ in range(num_shards)]
+        return json.load(fp)
 
-    def get_duration(file):
+
+def calculate_shards(all_files: List[str], num_shards: int = 20) -> List[List[str]]:
+    sharded_files: List[Tuple[float, List[str]]] = [(0.0, []) for _ in range(num_shards)]
+    metadata = read_metadata()
+
+    def get_duration(file: str) -> int:
         # tutorials not listed in the metadata.json file usually take
         # <3min to run, so we'll default to 1min if it's not listed
         return metadata.get(file, {}).get("duration", 60)
 
-    def get_needs_machine(file):
+    def get_needs_machine(file: str) -> Optional[str]:
         return metadata.get(file, {}).get("needs", None)
 
     def add_to_shard(i, filename):
@@ -55,9 +59,19 @@ def add_to_shard(i, filename):
     return [x[1] for x in sharded_files]
 
 
-def remove_other_files(all_files, files_to_run) -> None:
+def compute_files_to_keep(files_to_run: List[str]) -> List[str]:
+    metadata = read_metadata()
+    files_to_keep = list(files_to_run)
+    for file in files_to_run:
+        extra_files = metadata.get(file, {}).get("extra_files", [])
+        files_to_keep.extend(extra_files)
+    return files_to_keep
+
+
+def remove_other_files(all_files, files_to_keep) -> None:
+
     for file in all_files:
-        if file not in files_to_run:
+        if file not in files_to_keep:
             remove_runnable_code(file, file)
 
 
@@ -76,7 +90,7 @@ def main() -> None:
     all_files = get_all_files()
     files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num]
     if not args.dry_run:
-        remove_other_files(all_files, files_to_run)
+        remove_other_files(all_files, compute_files_to_keep(files_to_run))
     stripped_file_names = [Path(x).stem for x in files_to_run]
     print(" ".join(stripped_file_names))
 

.jenkins/metadata.json

@@ -1,4 +1,7 @@
 {
+  "intermediate_source/ax_multiobjective_nas_tutorial.py": {
+    "extra_files": ["intermediate_source/mnist_train_nas.py"]
+  },
   "beginner_source/dcgan_faces_tutorial.py": {
     "duration": 2000
   },

_static/img/ax_scheduler_illustration.png

@@ -850,7 +850,7 @@ def preprocess(x, y):
 #
 # We now have a general data pipeline and training loop which you can use for
 # training many types of models using Pytorch. To see how simple training a model
-# can now be, take a look at the `mnist_sample` sample notebook.
+# can now be, take a look at the `mnist_sample notebook <https://github.com/fastai/fastai_dev/blob/master/dev_nb/mnist_sample.ipynb>`__.
 #
 # Of course, there are many things you'll want to add, such as data augmentation,
 # hyperparameter tuning, monitoring training, transfer learning, and so forth.

_static/img/nvfuser_intro/nvfuser_transformer_block.png

@@ -38,6 +38,9 @@
 import distutils.file_util
 import re
 
+import plotly.io as pio
+pio.renderers.default = 'sphinx_gallery'
+
 
 try:
     import torchvision

_static/img/nvfuser_intro/nvfuser_tutorial_0.png

@@ -492,14 +492,28 @@ What's new in PyTorch tutorials?
    :image: _static/img/thumbnails/cropped/60-min-blitz.png
    :link: advanced/static_quantization_tutorial.html
    :tags: Quantization
-   
+
 .. customcarditem::
    :header: Grokking PyTorch Intel CPU Performance from First Principles
    :card_description: A case study on the TorchServe inference framework optimized with Intel® Extension for PyTorch.
    :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
    :link: intermediate/torchserve_with_ipex
    :tags: Model-Optimization,Production
 
+.. customcarditem::
+   :header: Introduction to nvFuser
+   :card_description: An introduction to nvFuser
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/nvfuser_intro_tutorial.html
+   :tags: Model-Optimization
+
+.. customcarditem::
+   :header: Multi-Objective Neural Architecture Search with Ax
+   :card_description: Learn how to use Ax to search over architectures find optimal tradeoffs between accuracy and latency.
+   :image: _static/img/ray-tune.png
+   :link: intermediate/ax_multiobjective_nas_tutorial.html
+   :tags: Model-Optimization,Best-Practice,Ax,TorchX
+
 .. Parallel-and-Distributed-Training
 
 .. customcarditem::
@@ -863,6 +877,8 @@ Additional Resources
    intermediate/quantized_transfer_learning_tutorial
    advanced/static_quantization_tutorial
    intermediate/torchserve_with_ipex
+   intermediate/nvfuser_intro_tutorial
+   intermediate/ax_multiobjective_nas_tutorial
 
 .. toctree::
    :maxdepth: 2
@@ -894,7 +910,7 @@ Additional Resources
 
    beginner/deeplabv3_on_ios
    beginner/deeplabv3_on_android
-   
+
 .. toctree::
    :maxdepth: 2
    :includehidden:

_static/img/nvfuser_intro/nvfuser_tutorial_1.png

@@ -32,3 +32,7 @@ Intermediate tutorials
 8. flask_rest_api_tutorial.py
 	Deploying PyTorch and Building a REST API using Flask
 	https://pytorch.org/tutorials/intermediate/flask_rest_api_tutorial.html
+
+9. nvfuser_intro_tutorial.py
+        Introduction to nvFuser
+	https://pytorch.org/tutorials/intermediate/nvfuser_intro_tutorial.html