Skip to content

Commit

Permalink
Improve the way the docs are published (#3030)
Browse files Browse the repository at this point in the history 
* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* restore publish

* wip

* remove

* wip

* minor changes

* minor changes
  • Loading branch information
@czoido
czoido authored Feb 28, 2023
1 parent 9554921 commit 000f2f2
Show file tree
Hide file tree
Showing 9 changed files with 412 additions and 56 deletions.
75 changes: 75 additions & 0 deletions .ci/new_publish.jenkins
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
#!groovy​


node('Linux') {

try {
String output_contents = 'output_contents'
String sources_folder = 'sources_folder'
String gh_pages_out = 'gh_pages_out'

checkout scm
def image = null
stage('Build docker image') {
// Build the docker image using the same commit as the current 'publish.jenkins' file
image = docker.build('conan-docs', '-f .ci/Dockerfile .') // It should cache the image
}

List branches = sh(script: 'python .ci/scripts/get_branches.py', returnStdout: true).trim().readLines()

stage('Prepare sources as worktrees') {
String branch_argument = ""
for (branch in branches) {
branch_argument = branch_argument + " --branches=${branch}"
}
// clone sources to generate docs
sh(script: "python .ci/scripts/prepare_sources.py --sources-folder=${sources_folder} ${branch_argument}")
}

// we have to divide the parallel blocks because if we have to generate all branches documentation
// it will fail

def number_of_parallel_blocks = 2
def branches_blocks = branches.collate(branches.size().intdiv(number_of_parallel_blocks))

for (branches_block in branches_blocks) {
Map parallelJobs = [:]
println("New block ${branches_block}")
for (branch in branches_block) {
parallelJobs[branch] = {
echo "Run parallel job for ${branch}"
image.inside {
sh(script: "python .ci/scripts/generate_documentation.py --sources-folder=${sources_folder} --branch=${branch}")
}
}
}
stage('Generate docs parallel block') {
parallelJobs.failFast = true
parallel parallelJobs
}
}

stage('Prepare gh-pages') {
sh(script: "python .ci/scripts/prepare_gh_pages.py --sources-folder=${sources_folder} --gh-pages-folder=${gh_pages_out}")
}


stage('Archive generated folder') {
archiveArtifacts artifacts: "${gh_pages_out}/**/*.*"
echo "Inspect generated webpage at ${BUILD_URL}artifact/${gh_pages_out}/index.html"
}

if (params.publish) {
stage('Publish to gh-pages') {
dir("${gh_pages_out}") {
sh 'ls'
}
}
}
}
finally {
cleanWs(cleanWhenAborted: true, cleanWhenFailure: true, cleanWhenNotBuilt: true,
cleanWhenSuccess: true, cleanWhenUnstable: true, disableDeferredWipeout: true, deleteDirs: true,
notFailBuild: true)
}
}
101 changes: 101 additions & 0 deletions .ci/scripts/common.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import os
import subprocess
from contextlib import contextmanager


latest_v2_folder = "2"
latest_v1_folder = "1"
latest_v1_branch = "master"

conan_versions = {
# the first of the dictionary
# must be always the latest version
"2.0": "release/2.0",
latest_v1_folder: latest_v1_branch,
"en/1.59": "release/1.59.0",
"en/1.58": "release/1.58.0",
"en/1.57": "release/1.57.0",
"en/1.56": "release/1.56.0",
"en/1.55": "release/1.55.0",
"en/1.54": "release/1.54.0",
"en/1.53": "release/1.53.0",
"en/1.52": "release/1.52.0",
"en/1.51": "release/1.51.3",
"en/1.50": "release/1.50.2",
"en/1.49": "release/1.49.0",
"en/1.48": "release/1.48.2",
"en/1.47": "release/1.47.0",
"en/1.46": "release/1.46.2",
"en/1.45": "release/1.45.0",
"en/1.44": "release/1.44.1",
"en/1.43": "release/1.43.4",
"en/1.42": "release/1.42.2",
"en/1.41": "release/1.41.0",
"en/1.40": "release/1.40.4",
"en/1.39": "release/1.39.0",
"en/1.38": "release/1.38.0",
"en/1.37": "release/1.37.2",
"en/1.36": "release/1.36.0",
"en/1.35": "release/1.35.2",
"en/1.34": "release/1.34.1",
"en/1.33": "release/1.33.1",
"en/1.32": "release/1.32.1",
"en/1.31": "release/1.31.4",
"en/1.30": "release/1.30.2",
"en/1.29": "release/1.29.2",
"en/1.28": "release/1.28.2",
"en/1.27": "release/1.27.1",
"en/1.26": "release/1.26.1",
"en/1.25": "release/1.25.2",
"en/1.24": "release/1.24.1",
"en/1.23": "release/1.23.0",
"en/1.22": "release/1.22.3",
"en/1.21": "release/1.21.3",
"en/1.20": "release/1.20.5",
"en/1.19": "release/1.19.3",
"en/1.18": "release/1.18.5",
"en/1.17": "release/1.17.2",
"en/1.16": "release/1.16.1",
"en/1.15": "release/1.15.2",
"en/1.14": "release/1.14.5",
"en/1.13": "release/1.13.3",
"en/1.12": "release/1.12.3",
"en/1.11": "release/1.11.2",
"en/1.10": "release/1.10.2",
"en/1.9": "release/1.9.4",
"en/1.8": "release/1.8.4",
"en/1.7": "release/1.7.4",
"en/1.6": "release/1.6.1",
"en/1.5": "release/1.5.2",
"en/1.4": "release/1.4.5",
"en/1.3": "release/1.3.3",
}

latest_v2_branch = list(conan_versions.values())[0]
latest_v2_version = list(conan_versions.keys())[0]


def run(cmd, capture=False):
stdout = subprocess.PIPE if capture else None
stderr = subprocess.PIPE if capture else None
process = subprocess.Popen(
cmd, stdout=stdout, stderr=stderr, shell=True)
out, err = process.communicate()
out = out.decode("utf-8") if capture else ""
err = err.decode("utf-8") if capture else ""
ret = process.returncode
output = err + out
if ret != 0:
raise Exception("Failed cmd: {}\n{}".format(cmd, output))
return output


@contextmanager
def chdir(dir_path):
current = os.getcwd()
os.makedirs(dir_path, exist_ok=True)
os.chdir(dir_path)
try:
yield
finally:
os.chdir(current)
45 changes: 45 additions & 0 deletions .ci/scripts/create_redirects.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import os
from pathlib import Path
import textwrap


def create_redirects(path_html, old_slug, new_slug):
"""
Redirect every html page found in sources_path from being located under the old_slug
subfolder to the new location in new_slug. For example if old_slug=en/latest and new_slug=1
docs.conan.io/en/latest/index.html --> redirects to --> docs.conan.io/1/index.html
"""

path_html = Path(path_html)

if not path_html.exists():
print("The html directory doesn't exist")
raise SystemExit(1)


def replace_html_files(sources_path: Path, old_slug: str, new_slug: str):

redirect_template = textwrap.dedent("""
<!DOCTYPE HTML>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="refresh" content="1; url={destination}">
</head>
</html>
""")

html_files = sources_path.glob('**/*.html')

for html_file in html_files:
origin = Path(old_slug) / Path(html_file).relative_to(
sources_path).parent
destination = Path(new_slug) / Path(html_file).relative_to(
sources_path)
redirect = Path(os.path.relpath(destination, origin))
with html_file.open('w') as f:
f.write(redirect_template.format(destination=redirect))


replace_html_files(path_html, old_slug, new_slug)
59 changes: 59 additions & 0 deletions .ci/scripts/generate_documentation.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import argparse
import json
import os

from common import chdir, conan_versions, latest_v2_folder, latest_v1_folder, latest_v2_branch, run

parser = argparse.ArgumentParser()

parser.add_argument("--branch", help="Docs branch to generate docs for", required=True)
parser.add_argument('--sources-folder',
help='Folder where the docs branches are cloned', required=True)
parser.add_argument('--with-pdf', default=False, action='store_true')

args = parser.parse_args()

branch = args.branch
with_pdf = args.with_pdf
sources_folder = args.sources_folder
output_folder = "output"

conan_versions[latest_v2_folder] = latest_v2_branch


branch_folder = [k for k, v in conan_versions.items() if v == branch][0]

print(f"branch_folder: {branch_folder}")

with chdir(f"{sources_folder}"):

with open(os.path.join(branch_folder, 'versions.json'), 'w') as versions_json:
json.dump(conan_versions, versions_json, indent=4)

if branch_folder != latest_v1_folder:
run(f"rm -fr {branch_folder}/_themes/conan")
run(f"cp -a {latest_v1_folder}/_themes/. {branch_folder}/_themes/")

# clone conan sources for autodoc
if branch_folder.startswith("2"):
# the branch in the docs for 2.0 has the same name that the one in Conan
conan_branch = branch
conan_repo_url = 'https://github.com/conan-io/conan.git'

# clone sources
run(f"rm -rf {branch_folder}/conan_sources")
run(f"git clone --single-branch -b {conan_branch} --depth 1 {conan_repo_url} {branch_folder}/conan_sources")

# for some reason even adding this to autodoc_mock_imports
# does not work, se we have to install the real dependency
# TODO: move this to jenkins
# run('pip3 install colorama')

# generate html
run(f"sphinx-build -W -b html -d {branch_folder}/_build/.doctrees {branch_folder}/ {output_folder}/{branch_folder}")

# generate pdf
if with_pdf:
run(f"sphinx-build -W -b latex -d {branch_folder}/_build/.doctrees {branch_folder}/ {branch_folder}/_build/latex")
run(f"make -C {branch_folder}/_build/latex all-pdf")
run(f"cp {branch_folder}/_build/latex/conan.pdf {output_folder}/{branch_folder}/conan.pdf")
29 changes: 29 additions & 0 deletions .ci/scripts/get_branches.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import os

from common import run, conan_versions


"""
Get the branches we have to build the docs for, there are two scenarios:
1. We changed something in the .ci scripts or change the _themes folder in the master
branch -> regenerate every branch of the docs.
2. If we did not touch those folders just regenerate the branch we pushed
"""
current_branch = os.getenv("BRANCH_NAME")

current_commit = run("git rev-parse HEAD", capture=True).strip()

previous_commit = run("git rev-parse HEAD^1", capture=True).strip()

diff = run(f"git diff --name-only {previous_commit}..{current_commit}", capture=True)

changed_ci = any([line.startswith(".ci") for line in diff.splitlines()])
changed_theme = any([line.startswith("_themes") for line in diff.splitlines()])

if not changed_ci and not (changed_theme and current_branch == "master"):
print(current_branch)
else:
for branch in conan_versions.values():
print(branch)
67 changes: 67 additions & 0 deletions .ci/scripts/prepare_gh_pages.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import os
import argparse
from pathlib import Path

from common import chdir, run, latest_v1_folder, latest_v2_folder, latest_v2_version
from create_redirects import create_redirects


parser = argparse.ArgumentParser()

parser.add_argument('--sources-folder',
help='Folder where the docs were created', required=True)

parser.add_argument('--gh-pages-folder',
help='Folder to clone the gh-pages branch to', required=True)

args = parser.parse_args()

output_folder = os.path.join(args.sources_folder, "output")
pages_folder = args.gh_pages_folder

with chdir(output_folder):
# FIXME: this is to not break all links from https://docs.conan.io/en/latest/
# we copy all the /1 folder to /en/latest and then replace all html files
# there with redirects to https://docs.conan.io/en/latest/1
# remove when most of the traffic in the docs is for 2.X docs

# First check if we generated any docs in `latest_v1_folder`
path_latest_v1 = Path(os.path.join(latest_v1_folder))
if path_latest_v1.exists():
run('mkdir -p en/latest')
run(f"cp -R {latest_v1_folder}/* en/latest")
create_redirects(path_html="en/latest", old_slug="en/latest", new_slug="1")

# 2 folder is the same as the latest 2.X, copy the generated html files to 2 folder
path_latest_v2 = Path(os.path.join(latest_v2_version))
if path_latest_v2.exists():
run(f"cp -R {latest_v2_version} {latest_v2_folder}")

#run(f"rm -rf {pages_folder}")

docs_repo_url = 'https://github.com/conan-io/docs.git'
run(f"git clone --single-branch -b gh-pages --depth 1 {docs_repo_url} {pages_folder}")

run(f"cp -R {output_folder}/* {pages_folder}")

run(f"cp {output_folder}/{latest_v2_folder}/404.html {pages_folder}/404.html")

path_404 = f"{pages_folder}/404.html"

with open(path_404, 'r') as file_404 :
contents_404 = file_404.read()

prefix = 'https://docs.conan.io'
prefix_latest = f"{prefix}/{latest_v2_folder}"

contents_404 = contents_404.replace('href="_', f"href=\"{prefix_latest}/_")
contents_404 = contents_404.replace('src="_', f"src=\"{prefix_latest}/_")
contents_404 = contents_404.replace('alt="_', f"alt=\"{prefix_latest}/_")
contents_404 = contents_404.replace('internal" href="', f"internal\" href=\"{prefix_latest}/")
contents_404 = contents_404.replace('"search.html"', f"\"{prefix_latest}/search.html\"")
contents_404 = contents_404.replace('"genindex.html"', f"\"{prefix_latest}/genindex.html\"")

with open(path_404, 'w') as file:
file.write(contents_404)

# gh-pages prepared to push
Loading

0 comments on commit 000f2f2

Please sign in to comment.