Reorg

dagster-io · Jan 23, 2025 · cae45d5 · cae45d5
1 parent 477b56c
commit cae45d5
Show file tree

Hide file tree

Showing 32 changed files with 42 additions and 62 deletions.
diff --git a/docs/docs-beta/docs/tutorials/category-one/first-tutorial/index.md b/docs/docs-beta/docs/tutorials/category-one/first-tutorial/index.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/first-tutorial/page-one.md b/docs/docs-beta/docs/tutorials/category-one/first-tutorial/page-one.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/first-tutorial/page-two.md b/docs/docs-beta/docs/tutorials/category-one/first-tutorial/page-two.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/second-tutorial/index.md b/docs/docs-beta/docs/tutorials/category-one/second-tutorial/index.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/second-tutorial/page-one.md b/docs/docs-beta/docs/tutorials/category-one/second-tutorial/page-one.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/second-tutorial/page-two.md b/docs/docs-beta/docs/tutorials/category-one/second-tutorial/page-two.md
diff --git a/docs/docs-beta/docs/tutorials/category-one/third-tutorial.md b/docs/docs-beta/docs/tutorials/category-one/third-tutorial.md
diff --git a/docs/docs-beta/docs/tutorials/category-two/first-tutorial/index.md b/docs/docs-beta/docs/tutorials/category-two/first-tutorial/index.md
diff --git a/docs/docs-beta/docs/tutorials/category-two/first-tutorial/page-one.md b/docs/docs-beta/docs/tutorials/category-two/first-tutorial/page-one.md
diff --git a/docs/docs-beta/docs/tutorials/category-two/first-tutorial/page-two.md b/docs/docs-beta/docs/tutorials/category-two/first-tutorial/page-two.md
diff --git a/docs/docs-beta/docs/tutorials/index.md b/docs/docs-beta/docs/tutorials/index.md
@@ -1,5 +1,4 @@
 ---
 title: Dagster tutorials
+sidebar_class_name: hidden
 ---
-
-TK
diff --git a/...wo/llm-fine-tuning/feature-engineering.md → ...ls/llm-fine-tuning/feature-engineering.md b/...wo/llm-fine-tuning/feature-engineering.md → ...ls/llm-fine-tuning/feature-engineering.md
diff --git a/...gory-two/llm-fine-tuning/file-creation.md → ...utorials/llm-fine-tuning/file-creation.md b/...gory-two/llm-fine-tuning/file-creation.md → ...utorials/llm-fine-tuning/file-creation.md
diff --git a/...als/category-two/llm-fine-tuning/index.md → ...a/docs/tutorials/llm-fine-tuning/index.md b/...als/category-two/llm-fine-tuning/index.md → ...a/docs/tutorials/llm-fine-tuning/index.md
@@ -31,7 +31,7 @@ To follow the steps in this guide, you'll need:
 
 First, set up a new Dagster project.
 
-1. Within the Dagster repo, navigate to the project:
+1. Clone the [Dagster repo](https://github.com/dagster-io/dagster) and navigate to the project:
 
    ```bash
    cd examples/dagster-llm-fine-tune

diff --git a/...category-two/llm-fine-tuning/ingestion.md → ...cs/tutorials/llm-fine-tuning/ingestion.md b/...category-two/llm-fine-tuning/ingestion.md → ...cs/tutorials/llm-fine-tuning/ingestion.md
diff --git a/...y-two/llm-fine-tuning/model-validation.md → ...rials/llm-fine-tuning/model-validation.md b/...y-two/llm-fine-tuning/model-validation.md → ...rials/llm-fine-tuning/model-validation.md
@@ -19,7 +19,3 @@ We will store the accuracy of both models as metadata in the check. Because this
 We can also execute this asset check separately from the fine-tuning job if we ever want to compare the accuracy. Running it a few more times, we can see that the accuracy is plotted:
 
 ![2048 resolution](/images/tutorial/llm-fine-tuning/model_accuracy_2.png)
-
-## Summary
-
-This should give you a good sense of how to fine-tune a model end to end, from ingesting the files, to creating features, and generating and validating the model.
diff --git a/...tegory-two/llm-fine-tuning/open-ai-job.md → .../tutorials/llm-fine-tuning/open-ai-job.md b/...tegory-two/llm-fine-tuning/open-ai-job.md → .../tutorials/llm-fine-tuning/open-ai-job.md
diff --git a/...ls/category-two/modal/factory-pipeline.md → .../docs/tutorials/modal/factory-pipeline.md b/...ls/category-two/modal/factory-pipeline.md → .../docs/tutorials/modal/factory-pipeline.md
diff --git a/...ocs/tutorials/category-two/modal/index.md → docs/docs-beta/docs/tutorials/modal/index.md b/...ocs/tutorials/category-two/modal/index.md → docs/docs-beta/docs/tutorials/modal/index.md
diff --git a/...s/category-two/modal/modal-application.md → ...docs/tutorials/modal/modal-application.md b/...s/category-two/modal/modal-application.md → ...docs/tutorials/modal/modal-application.md
diff --git a/...utorials/category-two/modal/rss-assets.md → ...s-beta/docs/tutorials/modal/rss-assets.md b/...utorials/category-two/modal/rss-assets.md → ...s-beta/docs/tutorials/modal/rss-assets.md
diff --git a/...o/prompt-engineering/additional-prompt.md → ...s/prompt-engineering/additional-prompt.md b/...o/prompt-engineering/additional-prompt.md → ...s/prompt-engineering/additional-prompt.md
diff --git a/...two/prompt-engineering/custom-resource.md → ...als/prompt-engineering/custom-resource.md b/...two/prompt-engineering/custom-resource.md → ...als/prompt-engineering/custom-resource.md
diff --git a/.../category-two/prompt-engineering/index.md → ...ocs/tutorials/prompt-engineering/index.md b/.../category-two/prompt-engineering/index.md → ...ocs/tutorials/prompt-engineering/index.md
diff --git a/...ategory-two/prompt-engineering/prompts.md → ...s/tutorials/prompt-engineering/prompts.md b/...ategory-two/prompt-engineering/prompts.md → ...s/tutorials/prompt-engineering/prompts.md
diff --git a/.../tutorials/category-two/rag/embeddings.md → ...ocs-beta/docs/tutorials/rag/embeddings.md b/.../tutorials/category-two/rag/embeddings.md → ...ocs-beta/docs/tutorials/rag/embeddings.md
@@ -32,7 +32,7 @@ This I/O manager will be attached to the <PyObject section="definitions" module=
 
 ## Scraping embeddings
 
-The assets for the documentation scraping will behave similar to the GitHub assets. We do not need to worry about rate limiting in the same way as GitHub, so we can leave out the partition that we had defined for GitHub. Instead, we will just include half a second sleep between scraping pages. But like the GitHub assets, our ingestion asset will return a collection of `Documents` that will be handled by the I/O manager. This asset will also include the <PyObject section="assets" module="dagster" object="AutomationCondition" /> to update data on the same cadence as our GitHub source.
+The assets for the documentation scraping will behave similar to the GitHub assets. We will not partition by date like Github, so we can leave out that out of the asset. But like the GitHub assets, our ingestion asset will return a collection of `Documents` that will be handled by the I/O manager. This asset will also include the <PyObject section="assets" module="dagster" object="AutomationCondition" /> to update data on the same cadence as our GitHub source.
 
 The asset that generates the embeddings with the documentation site will need one additional change. Because the content of the documentation pages is so large, we need to split data into chunks. The `split_text` function ensures that we split the text into equal length chunks. We also want to keep similar chunks together and associated with the page they were on so we will hash the index of the URL to ensure data stays together. correctly Once the data is chunked, it can be batched and sent to Pinecone:
 

diff --git a/.../docs/tutorials/category-two/rag/index.md → docs/docs-beta/docs/tutorials/rag/index.md b/.../docs/tutorials/category-two/rag/index.md → docs/docs-beta/docs/tutorials/rag/index.md
diff --git a/...s/tutorials/category-two/rag/retrieval.md → ...docs-beta/docs/tutorials/rag/retrieval.md b/...s/tutorials/category-two/rag/retrieval.md → ...docs-beta/docs/tutorials/rag/retrieval.md
diff --git a/...ocs/tutorials/category-two/rag/sources.md → docs/docs-beta/docs/tutorials/rag/sources.md b/...ocs/tutorials/category-two/rag/sources.md → docs/docs-beta/docs/tutorials/rag/sources.md
diff --git a/...rials/category-two/rag/vector-database.md → ...eta/docs/tutorials/rag/vector-database.md b/...rials/category-two/rag/vector-database.md → ...eta/docs/tutorials/rag/vector-database.md
diff --git a/docs/docs-beta/sidebars.ts b/docs/docs-beta/sidebars.ts
@@ -124,23 +124,45 @@ const sidebars: SidebarsConfig = {
     'tutorials/index',
     {
       type: 'category',
-      label: 'Category one',
-      collapsed: false,
+      label: 'RAG',
+      collapsed: true,
       items: [
         {
           type: 'autogenerated',
-          dirName: 'tutorials/category-one',
+          dirName: 'tutorials/rag',
         },
       ],
     },
     {
       type: 'category',
-      label: 'Category two',
-      collapsed: false,
+      label: 'Fine-tuning',
+      collapsed: true,
+      items: [
+        {
+          type: 'autogenerated',
+          dirName: 'tutorials/llm-fine-tuning',
+        },
+      ],
+    },
+    {
+      type: 'category',
+      label: 'Prompt engineering',
+      collapsed: true,
+      items: [
+        {
+          type: 'autogenerated',
+          dirName: 'tutorials/prompt-engineering',
+        },
+      ],
+    },
+    {
+      type: 'category',
+      label: 'Modal',
+      collapsed: true,
       items: [
         {
           type: 'autogenerated',
-          dirName: 'tutorials/category-two',
+          dirName: 'tutorials/modal',
         },
       ],
     },

diff --git a/docs/docs-beta/src/code-examples-content.js b/docs/docs-beta/src/code-examples-content.js