Improving documentation CI workflows

.github/workflows/build-and-test-on-pr.yml

@@ -15,13 +15,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Use Node.js
-        uses: actions/setup-node@v2
+      - uses: actions/setup-node@v2
         with:
           node-version: '14.x'
-
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
@@ -30,7 +26,9 @@ jobs:
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
           key: ${{ runner.os }}-yarn-2-${{ hashFiles('**/package.json') }}
-
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
       - run: yarn install --frozen-lockfile
       - run: yarn bootstrap
       - run: yarn build
@@ -42,5 +40,6 @@ jobs:
           POSTGRES_PASSWORD: test123
           POSTGRES_PORT: 5432
         run: yarn test:integration
+      - run: yarn docs
       - run: yarn lint
       - run: npx codecov

.github/workflows/build-test-publish-on-push.yml

@@ -53,7 +53,8 @@ jobs:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: test123
           POSTGRES_PORT: 5432
-        run: yarn test:integration
+        run: yarn test:integration        
+      - run: yarn docs
 
       - name: setup git coordinates
         run: |

.github/workflows/veramo-website.yml

@@ -0,0 +1,20 @@
+name: Trigger veramo-website build
+on:
+  push:
+    branches:
+      - "main"
+      - "italo/fix-docs"
+permissions: write-all
+jobs:
+  dispatch:
+    permissions:
+      contents: write
+      actions: write 
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          curl -X POST \
+            -H 'Authorization: Bearer ${{ secrets.GH_TOKEN }}' \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/uport-project/veramo-website/actions/workflows/main.yml/dispatches \
+            -d '{"ref":"main"}'

package.json

@@ -14,19 +14,25 @@
     "test": "jest --config=jest.json --coverage=false",
     "test:watch": "yarn test --watch --verbose",
     "veramo": "./packages/cli/bin/veramo.js",
-    "prettier": "prettier --write '{packages,__tests__, !build}/**/*.{ts,js,json,md,yml}'",
+    "prettier": "prettier --write '{packages,docs,__tests__, !build}/**/*.{ts,js,json,md,yml}'",
     "build-clean": "rimraf ./packages/*/build ./packages/*/api ./packages/*/node_modules ./packages/*/tsconfig.tsbuildinfo && jest --clearCache",
     "publish:latest": "lerna publish --conventional-commits --include-merged-tags --create-release github --yes --dist-tag latest --registry https://registry.npmjs.org/:_authToken=${NPM_TOKEN}",
     "publish:next": "lerna publish --conventional-prerelease --force-publish --canary --no-git-tag-version --include-merged-tags --preid next --pre-dist-tag next --yes --registry https://registry.npmjs.org/:_authToken=${NPM_TOKEN}",
-    "publish:unstable": "lerna publish --conventional-prerelease --force-publish --canary --no-git-tag-version --include-merged-tags --preid unstable --pre-dist-tag unstable --yes --registry https://registry.npmjs.org/:_authToken=${NPM_TOKEN}"
+    "publish:unstable": "lerna publish --conventional-prerelease --force-publish --canary --no-git-tag-version --include-merged-tags --preid unstable --pre-dist-tag unstable --yes --registry https://registry.npmjs.org/:_authToken=${NPM_TOKEN}",
+    "docs": "yarn docs:extract && yarn docs:copy && yarn docs:build",
+    "docs:extract": "ts-node ./scripts/docs-extract.ts",
+    "docs:copy": "ts-node ./scripts/docs-copy.ts",
+    "docs:build": "ts-node ./scripts/docs-build.ts && yarn docs:pretty",    
+    "docs:pretty": "pretty-quick"
   },
   "workspaces": [
     "packages/*"
   ],
-  "devDependencies": {
-    "@microsoft/api-documenter": "7.13.68",
-    "@microsoft/api-extractor": "7.18.19",
-    "@microsoft/api-extractor-model": "7.13.16",
+  "devDependencies": {    
+    "@microsoft/api-documenter": "^7.17.17",
+    "@microsoft/api-extractor": "^7.25.0",
+    "@microsoft/api-extractor-model": "^7.18.0",
+    "@types/fs-extra": "^9.0.13",
     "@types/jest": "27.0.2",
     "codecov": "3.8.3",
     "cross-fetch": "3.1.4",
@@ -44,9 +50,9 @@
     "rimraf": "3.0.2",
     "semantic-release": "18.0.0",
     "ts-jest": "27.0.7",
-    "ts-json-schema-generator": "0.97.0",
-    "ts-node": "10.4.0",
-    "typescript": "4.4.4"
+    "ts-json-schema-generator": "1.0.0",
+    "ts-node": "10.8.0",
+    "typescript": "4.6.3"
   },
   "repository": {
     "type": "git",

packages/cli/src/dev.ts

@@ -110,7 +110,7 @@ dev
     const packageConfig = require(resolve(options.packageConfig))
     const interfaces: any = {}
 
-    for (const pluginInterfaceName in packageConfig.veramo.pluginInterfaces) {
+    for (const pluginInterfaceName in packageConfig?.veramo?.pluginInterfaces) {
       const entryFile = packageConfig.veramo.pluginInterfaces[pluginInterfaceName]
       const api = {
         components: {

scripts/docs-build.ts

@@ -0,0 +1,104 @@
+import { readdir, createReadStream, writeFile } from 'fs-extra'
+import { createInterface } from 'readline'
+import { join, parse } from 'path'
+import { exec } from 'child_process'
+import * as fs from 'fs'
+
+const DOCS_DIR = './docs/api'
+
+async function main() {
+  await fs.promises.mkdir(DOCS_DIR, { recursive: true })
+
+  await new Promise((resolve, reject) =>
+    exec(`api-documenter markdown -i ./temp -o ${DOCS_DIR}`, (err, stdout, stderr) => {
+      console.log(stdout)
+      console.error(stderr)
+      if (err) {
+        reject(err)
+      } else {
+        resolve('')
+      }
+    }),
+  )
+
+  const docFiles = await readdir(DOCS_DIR)
+  for (const docFile of docFiles) {
+    try {
+      const { name: id, ext } = parse(docFile)
+      if (ext !== '.md') {
+        continue
+      }
+
+      const docPath = join(DOCS_DIR, docFile)
+      const input = createReadStream(docPath)
+      const output: string[] = []
+      const lines = createInterface({
+        input,
+        crlfDelay: Infinity,
+      })
+
+      let title = ''
+      lines.on('line', (line) => {
+        let skip = false
+        if (!title) {
+          const titleLine = line.match(/## (.*)/)
+          if (titleLine) {
+            title = titleLine[1]
+          }
+        }
+        const indexHomeLink = line.match(/\[Home\]\(.\/index\.md\)/)
+        const homeLink = line.match(/\[Home\]\(.\/index\.md\) > (.*)/)
+        if (homeLink) {
+          line = line.replace('Home', 'Packages')
+        }
+
+        if (indexHomeLink) {
+          // Skip the breadcrumb for the toplevel index file.
+          if (id === 'index') {
+            skip = true
+          }
+
+          skip = true
+        }
+
+        // See issue #4. api-documenter expects \| to escape table
+        // column delimiters, but docusaurus uses a markdown processor
+        // that doesn't support this. Replace with an escape sequence
+        // that renders |.
+        if (line.startsWith('|')) {
+          line = line.replace(/\\\|/g, '|')
+        }
+
+        // MDX cries when you put commects in there :(
+        line = replaceAll(line, '<!-- -->', '')
+
+        if (id === 'core') {
+          line = line.replace('core package', 'Veramo Core')
+        }
+
+        if (!skip) {
+          output.push(line)
+        }
+      })
+
+      await new Promise((resolve) => lines.once('close', resolve))
+      input.close()
+
+      const header = ['---', `id: ${id}`, `title: ${title}`, `hide_title: true`, '---']
+
+      await writeFile(docPath, header.concat(output).join('\n'))
+    } catch (err) {
+      console.error(`Could not process ${docFile}: ${err}`)
+    }
+  }
+}
+
+function escapeRegExp(string: string) {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') // $& means the whole matched string
+}
+
+function replaceAll(str: string, find: string, replace: string) {
+  return str.replace(new RegExp(escapeRegExp(find), 'g'), replace)
+}
+
+main()

scripts/docs-copy.ts

@@ -0,0 +1,25 @@
+import * as path from 'path'
+import { resolve } from 'path'
+import { existsSync, readdirSync, copyFileSync, mkdirSync, unlinkSync } from 'fs'
+
+const outputFolder = './temp'
+const { documentPackages } = require('../docsconfig.json')
+
+if (!existsSync(resolve(outputFolder))) {
+  console.log('Creating', outputFolder)
+  mkdirSync(resolve(outputFolder))
+} else {
+  console.log('Removing files in', outputFolder)
+  readdirSync(resolve(outputFolder)).forEach((file) => {
+    unlinkSync(resolve(outputFolder, file))
+  })
+}
+
+for (const packageName of documentPackages) {
+  const apiDocsPath: string = path.join(__dirname, `../packages/${packageName}/api`)
+
+  readdirSync(apiDocsPath).forEach((file) => {
+    console.log('Copying', resolve(outputFolder, file))
+    copyFileSync(resolve(apiDocsPath, file), resolve(outputFolder, file))
+  })
+}

scripts/docs-extract.ts

@@ -0,0 +1,30 @@
+import * as path from 'path'
+import { Extractor, ExtractorConfig, ExtractorResult } from '@microsoft/api-extractor'
+
+const { documentPackages } = require('../docsconfig.json')
+
+documentPackages.map((packageName: string) => {
+  const apiExtractorJsonPath: string = path.join(__dirname, `../packages/${packageName}/api-extractor.json`)
+  // Load and parse the api-extractor.json file
+  const extractorConfig: ExtractorConfig = ExtractorConfig.loadFileAndPrepare(apiExtractorJsonPath)
+
+  // Invoke API Extractor
+  const extractorResult: ExtractorResult = Extractor.invoke(extractorConfig, {
+    // Equivalent to the "--local" command-line parameter
+    localBuild: true,
+
+    // Equivalent to the "--verbose" command-line parameter
+    showVerboseMessages: true,
+  })
+
+  if (extractorResult.succeeded) {
+    console.log(`API Extractor completed successfully`)
+    process.exitCode = 0
+  } else {
+    console.error(
+      `API Extractor completed with ${extractorResult.errorCount} errors` +
+        ` and ${extractorResult.warningCount} warnings`,
+    )
+    process.exitCode = 1
+  }
+})