scripts[minor]: Add notebook validation script (#6541)

* scripts[minor]: Add notebook validation script * trigger ntbk change * cr * format and lint * add lg=ogs * verify file exists * fixed * add -- to script * release 0.0.21 * bump all langchain scripts versions * Apply suggestions from code review * merge main
langchain-ai · Aug 16, 2024 · e378d9d · e378d9d
1 parent 3d2aaba
commit e378d9d
Show file tree

Hide file tree

Showing 39 changed files with 81 additions and 78 deletions.
diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml
@@ -51,8 +51,9 @@ jobs:
           notebooks=$(echo '${{ steps.changed-files.outputs.all_changed_files }}' | tr ' ' '\n' | grep '^docs/core_docs/.*\.ipynb$' || true)
           if [ -n "$notebooks" ]; then
             for notebook in $notebooks; do
-              yarn notebook:validate "$notebook"
+              absolute_path="$GITHUB_WORKSPACE/$notebook"
+              yarn notebook:validate "$absolute_path"
             done
           else
             echo "No notebooks in docs/core_docs to validate."
-          fi
+          fi
diff --git a/docs/core_docs/package.json b/docs/core_docs/package.json
@@ -27,7 +27,7 @@
     "broken-links": "node ./scripts/check-broken-links.js",
     "check:broken-links": "yarn quarto && yarn broken-links",
     "check:broken-links:ci": "yarn quarto:vercel && yarn broken-links",
-    "validate": "tsx --experimental-wasm-modules -r dotenv/config ./scripts/validate_notebook.ts"
+    "validate": "yarn notebook_validate"
   },
   "dependencies": {
     "@docusaurus/core": "2.4.3",

diff --git a/examples/package.json b/examples/package.json
@@ -58,7 +58,7 @@
     "@langchain/pinecone": "workspace:*",
     "@langchain/qdrant": "workspace:*",
     "@langchain/redis": "workspace:*",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/textsplitters": "workspace:*",
     "@langchain/weaviate": "workspace:*",
     "@langchain/yandex": "workspace:*",

diff --git a/langchain-core/package.json b/langchain-core/package.json
@@ -55,7 +55,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@types/decamelize": "^1.2.0",

diff --git a/langchain/package.json b/langchain/package.json
@@ -621,7 +621,7 @@
     "@langchain/groq": "^0.0.15",
     "@langchain/mistralai": "^0.0.26",
     "@langchain/ollama": "^0.0.2",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@mendable/firecrawl-js": "^0.0.13",
     "@notionhq/client": "^2.2.10",
     "@pinecone-database/pinecone": "^1.1.0",

diff --git a/libs/create-langchain-integration/template/package.json b/libs/create-langchain-integration/template/package.json
@@ -45,7 +45,7 @@
     "@jest/globals": "^29.5.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@tsconfig/recommended": "^1.0.3",
     "@typescript-eslint/eslint-plugin": "^6.12.0",
     "@typescript-eslint/parser": "^6.12.0",

diff --git a/libs/langchain-anthropic/package.json b/libs/langchain-anthropic/package.json
@@ -43,7 +43,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-aws/package.json b/libs/langchain-aws/package.json
@@ -50,7 +50,7 @@
   "devDependencies": {
     "@aws-sdk/types": "^3.609.0",
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@smithy/types": "^3.2.0",
     "@swc/core": "^1.3.90",

diff --git a/libs/langchain-azure-dynamic-sessions/package.json b/libs/langchain-azure-dynamic-sessions/package.json
@@ -44,7 +44,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-azure-openai/package.json b/libs/langchain-azure-openai/package.json
@@ -41,7 +41,7 @@
   "devDependencies": {
     "@azure/identity": "^4.2.1",
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-baidu-qianfan/package.json b/libs/langchain-baidu-qianfan/package.json
@@ -40,7 +40,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "~0.1.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-cloudflare/package.json b/libs/langchain-cloudflare/package.json
@@ -43,7 +43,7 @@
     "@cloudflare/workers-types": "^4.20231218.0",
     "@jest/globals": "^29.5.0",
     "@langchain/langgraph": "~0.0.31",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-cohere/package.json b/libs/langchain-cohere/package.json
@@ -43,7 +43,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json
@@ -79,7 +79,7 @@
     "@huggingface/inference": "^2.6.4",
     "@jest/globals": "^29.5.0",
     "@langchain/langgraph": "<0.1.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@layerup/layerup-security": "^1.5.12",
     "@mendable/firecrawl-js": "^0.0.36",

diff --git a/libs/langchain-exa/package.json b/libs/langchain-exa/package.json
@@ -45,7 +45,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-google-common/package.json b/libs/langchain-google-common/package.json
@@ -46,7 +46,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json
@@ -41,7 +41,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json
@@ -41,7 +41,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-google-vertexai-web/package.json b/libs/langchain-google-vertexai-web/package.json
@@ -45,7 +45,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-google-vertexai/package.json b/libs/langchain-google-vertexai/package.json
@@ -49,7 +49,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/google-common": "~0.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-google-webauth/package.json b/libs/langchain-google-webauth/package.json
@@ -46,7 +46,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-groq/package.json b/libs/langchain-groq/package.json
@@ -44,7 +44,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "workspace:^",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-mistralai/package.json b/libs/langchain-mistralai/package.json
@@ -43,7 +43,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-mixedbread-ai/package.json b/libs/langchain-mixedbread-ai/package.json
@@ -40,7 +40,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-mongodb/package.json b/libs/langchain-mongodb/package.json
@@ -46,7 +46,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "workspace:*",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-nomic/package.json b/libs/langchain-nomic/package.json
@@ -46,7 +46,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "workspace:^",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-ollama/package.json b/libs/langchain-ollama/package.json
@@ -41,7 +41,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-openai/package.json b/libs/langchain-openai/package.json
@@ -44,7 +44,7 @@
   "devDependencies": {
     "@azure/identity": "^4.2.1",
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",

diff --git a/libs/langchain-pinecone/package.json b/libs/langchain-pinecone/package.json
@@ -48,7 +48,7 @@
     "@faker-js/faker": "^8.3.1",
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "workspace:*",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-qdrant/package.json b/libs/langchain-qdrant/package.json
@@ -46,7 +46,7 @@
   "devDependencies": {
     "@faker-js/faker": "^8.4.1",
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-redis/package.json b/libs/langchain-redis/package.json
@@ -45,7 +45,7 @@
   "devDependencies": {
     "@faker-js/faker": "^8.4.0",
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-scripts/bin/validate_notebook.js b/libs/langchain-scripts/bin/validate_notebook.js
@@ -0,0 +1 @@
+import "../dist/validate_notebook.js";
diff --git a/libs/langchain-scripts/package.json b/libs/langchain-scripts/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@langchain/scripts",
-  "version": "0.0.20",
+  "version": "0.0.21",
   "description": "Shared scripts for LangChain.js",
   "type": "module",
   "engines": {
@@ -15,7 +15,8 @@
   "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-scripts/",
   "bin": {
     "lc-build": "bin/build.js",
-    "lc_build_v2": "bin/build_v2.js"
+    "lc_build_v2": "bin/build_v2.js",
+    "notebook_validate": "bin/validate_notebook.js"
   },
   "scripts": {
     "build": "yarn turbo:command build:internal --filter=@langchain/scripts",

diff --git a/docs/core_docs/scripts/validate_notebook.ts → ...angchain-scripts/src/validate_notebook.ts b/docs/core_docs/scripts/validate_notebook.ts → ...angchain-scripts/src/validate_notebook.ts
@@ -1,15 +1,16 @@
-import * as fs from "node:fs";
-import * as ts from "typescript";
+import fs from "node:fs";
+import ts from "typescript";
 import { Project } from "ts-morph";
 
 export function extract(filepath: string) {
-  const cells = JSON.parse(fs.readFileSync(filepath).toString()).cells;
+  const { cells } = JSON.parse(fs.readFileSync(filepath).toString());
   if (cells[0]?.source.includes("lc_docs_skip_validation: true\n")) {
     return "";
   }
   const project = new Project({ useInMemoryFileSystem: true });
   const sourceFile = project.createSourceFile("temp.ts", "");
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   cells.forEach((cell: Record<string, any>) => {
     const source = cell.source
       .join("")
@@ -62,16 +63,13 @@ export function extract(filepath: string) {
   return sourceFile.getFullText();
 }
 
-let [pathname, ...args] = process.argv.slice(2);
+const [pathname] = process.argv.slice(2);
 
 if (!pathname) {
   throw new Error("No pathname provided.");
 }
 
 const run = async () => {
-  if (pathname.startsWith("docs/core_docs/")) {
-    pathname = "./" + pathname.slice("docs/core_docs/".length);
-  }
   if (!pathname.endsWith(".ipynb")) {
     throw new Error("Only .ipynb files are supported.");
   }
@@ -121,6 +119,7 @@ const run = async () => {
       const issues = issueStrings.join("\n");
       console.error(issues);
       const err = new Error("Found type errors in new notebook.");
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
       (err as any).details = issues;
       throw err;
     }
@@ -134,7 +133,7 @@ const run = async () => {
 };
 
 try {
-  run();
+  void run();
 } catch {
   process.exit(1);
 }
diff --git a/libs/langchain-textsplitters/package.json b/libs/langchain-textsplitters/package.json
@@ -44,7 +44,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-weaviate/package.json b/libs/langchain-weaviate/package.json
@@ -47,7 +47,7 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@langchain/openai": "workspace:^",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/libs/langchain-yandex/package.json b/libs/langchain-yandex/package.json
@@ -44,7 +44,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/scripts": "~0.0.20",
+    "@langchain/scripts": "^0.0.21",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",
     "@tsconfig/recommended": "^1.0.3",

diff --git a/package.json b/package.json
@@ -37,7 +37,7 @@
     "test:standard:int": "turbo test:standard:int",
     "test:standard": "yarn test:standard:unit && yarn test:standard:int",
     "example": "yarn workspace examples start",
-    "notebook:validate": "yarn workspace core_docs validate",
+    "notebook:validate": "turbo --filter=@langchain/scripts notebook:validate --",
     "precommit": "turbo precommit",
     "docs": "yarn workspace core_docs start",
     "docs:api_refs": "yarn workspace api_refs start",