Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

scripts[minor]: Add notebook validation script #6541

Merged
merged 13 commits into from
Aug 16, 2024
Merged

scripts[minor]: Add notebook validation script #6541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
bb913c3
scripts[minor]: Add notebook validation script
bracesproul Aug 15, 2024
ba34cfe
trigger ntbk change
bracesproul Aug 15, 2024
bf4466e
cr
bracesproul Aug 15, 2024
646869a
format and lint
bracesproul Aug 15, 2024
fe597a4
add lg=ogs
bracesproul Aug 15, 2024
76c3674
verify file exists
bracesproul Aug 16, 2024
afff7f2
fixed
bracesproul Aug 16, 2024
c7173b9
add -- to script
bracesproul Aug 16, 2024
d2be388
release 0.0.21
bracesproul Aug 16, 2024
8af9746
bump all langchain scripts versions
bracesproul Aug 16, 2024
ce9cc0d
Apply suggestions from code review
bracesproul Aug 16, 2024
d2bec1c
merge main
bracesproul Aug 16, 2024
26dbac9
merge main
bracesproul Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/validate_new_notebooks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ jobs:
notebooks=$(echo '${{ steps.changed-files.outputs.all_changed_files }}' | tr ' ' '\n' | grep '^docs/core_docs/.*\.ipynb$' || true)
if [ -n "$notebooks" ]; then
for notebook in $notebooks; do
yarn notebook:validate "$notebook"
absolute_path="$GITHUB_WORKSPACE/$notebook"
yarn notebook:validate "$absolute_path"
done
else
echo "No notebooks in docs/core_docs to validate."
fi
fi
2 changes: 1 addition & 1 deletion docs/core_docs/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"broken-links": "node ./scripts/check-broken-links.js",
"check:broken-links": "yarn quarto && yarn broken-links",
"check:broken-links:ci": "yarn quarto:vercel && yarn broken-links",
"validate": "tsx --experimental-wasm-modules -r dotenv/config ./scripts/validate_notebook.ts"
"validate": "yarn notebook_validate"
},
"dependencies": {
"@docusaurus/core": "2.4.3",
Expand Down
2 changes: 1 addition & 1 deletion examples/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
"@langchain/pinecone": "workspace:*",
"@langchain/qdrant": "workspace:*",
"@langchain/redis": "workspace:*",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/textsplitters": "workspace:*",
"@langchain/weaviate": "workspace:*",
"@langchain/yandex": "workspace:*",
Expand Down
2 changes: 1 addition & 1 deletion langchain-core/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@types/decamelize": "^1.2.0",
Expand Down
2 changes: 1 addition & 1 deletion langchain/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@
"@langchain/groq": "^0.0.15",
"@langchain/mistralai": "^0.0.26",
"@langchain/ollama": "^0.0.2",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@mendable/firecrawl-js": "^0.0.13",
"@notionhq/client": "^2.2.10",
"@pinecone-database/pinecone": "^1.1.0",
Expand Down
2 changes: 1 addition & 1 deletion libs/create-langchain-integration/template/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"@jest/globals": "^29.5.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@tsconfig/recommended": "^1.0.3",
"@typescript-eslint/eslint-plugin": "^6.12.0",
"@typescript-eslint/parser": "^6.12.0",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-anthropic/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-aws/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"devDependencies": {
"@aws-sdk/types": "^3.609.0",
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@smithy/types": "^3.2.0",
"@swc/core": "^1.3.90",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-azure-dynamic-sessions/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-azure-openai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"devDependencies": {
"@azure/identity": "^4.2.1",
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-baidu-qianfan/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/openai": "~0.1.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-cloudflare/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"@cloudflare/workers-types": "^4.20231218.0",
"@jest/globals": "^29.5.0",
"@langchain/langgraph": "~0.0.31",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-cohere/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-community/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
"@huggingface/inference": "^2.6.4",
"@jest/globals": "^29.5.0",
"@langchain/langgraph": "<0.1.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@layerup/layerup-security": "^1.5.12",
"@mendable/firecrawl-js": "^0.0.36",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-exa/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-common/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-gauth/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-genai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-vertexai-web/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-vertexai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/google-common": "~0.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-google-webauth/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-groq/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/openai": "workspace:^",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-mistralai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-mixedbread-ai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-mongodb/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/openai": "workspace:*",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-nomic/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/openai": "workspace:^",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-ollama/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-openai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"devDependencies": {
"@azure/identity": "^4.2.1",
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@langchain/standard-tests": "0.0.0",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-pinecone/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"@faker-js/faker": "^8.3.1",
"@jest/globals": "^29.5.0",
"@langchain/openai": "workspace:*",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-qdrant/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"devDependencies": {
"@faker-js/faker": "^8.4.1",
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-redis/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"devDependencies": {
"@faker-js/faker": "^8.4.0",
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
1 change: 1 addition & 0 deletions libs/langchain-scripts/bin/validate_notebook.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
import "../dist/validate_notebook.js";
5 changes: 3 additions & 2 deletions libs/langchain-scripts/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@langchain/scripts",
"version": "0.0.20",
"version": "0.0.21",
"description": "Shared scripts for LangChain.js",
"type": "module",
"engines": {
Expand All @@ -15,7 +15,8 @@
"homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-scripts/",
"bin": {
"lc-build": "bin/build.js",
"lc_build_v2": "bin/build_v2.js"
"lc_build_v2": "bin/build_v2.js",
"notebook_validate": "bin/validate_notebook.js"
},
"scripts": {
"build": "yarn turbo:command build:internal --filter=@langchain/scripts",
Expand Down
15 changes: 7 additions & 8 deletions docs/core_docs/scripts/validate_notebook.ts → ...angchain-scripts/src/validate_notebook.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
import * as fs from "node:fs";
import * as ts from "typescript";
import fs from "node:fs";
import ts from "typescript";
import { Project } from "ts-morph";

export function extract(filepath: string) {
const cells = JSON.parse(fs.readFileSync(filepath).toString()).cells;
const { cells } = JSON.parse(fs.readFileSync(filepath).toString());
if (cells[0]?.source.includes("lc_docs_skip_validation: true\n")) {
return "";
}
const project = new Project({ useInMemoryFileSystem: true });
const sourceFile = project.createSourceFile("temp.ts", "");

// eslint-disable-next-line @typescript-eslint/no-explicit-any
cells.forEach((cell: Record<string, any>) => {
const source = cell.source
.join("")
Expand Down Expand Up @@ -62,16 +63,13 @@ export function extract(filepath: string) {
return sourceFile.getFullText();
}

let [pathname, ...args] = process.argv.slice(2);
const [pathname] = process.argv.slice(2);

if (!pathname) {
throw new Error("No pathname provided.");
}

const run = async () => {
if (pathname.startsWith("docs/core_docs/")) {
pathname = "./" + pathname.slice("docs/core_docs/".length);
}
if (!pathname.endsWith(".ipynb")) {
throw new Error("Only .ipynb files are supported.");
}
Expand Down Expand Up @@ -121,6 +119,7 @@ const run = async () => {
const issues = issueStrings.join("\n");
console.error(issues);
const err = new Error("Found type errors in new notebook.");
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(err as any).details = issues;
throw err;
}
Expand All @@ -134,7 +133,7 @@ const run = async () => {
};

try {
run();
void run();
} catch {
process.exit(1);
}
2 changes: 1 addition & 1 deletion libs/langchain-textsplitters/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-weaviate/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/openai": "workspace:^",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion libs/langchain-yandex/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
},
"devDependencies": {
"@jest/globals": "^29.5.0",
"@langchain/scripts": "~0.0.20",
"@langchain/scripts": "^0.0.21",
"@swc/core": "^1.3.90",
"@swc/jest": "^0.2.29",
"@tsconfig/recommended": "^1.0.3",
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"test:standard:int": "turbo test:standard:int",
"test:standard": "yarn test:standard:unit && yarn test:standard:int",
"example": "yarn workspace examples start",
"notebook:validate": "yarn workspace core_docs validate",
"notebook:validate": "turbo --filter=@langchain/scripts notebook:validate --",
"precommit": "turbo precommit",
"docs": "yarn workspace core_docs start",
"docs:api_refs": "yarn workspace api_refs start",
Expand Down
Loading
Loading