diff --git a/.ci/new_publish.jenkins b/.ci/new_publish.jenkins new file mode 100644 index 000000000000..237c821a2347 --- /dev/null +++ b/.ci/new_publish.jenkins @@ -0,0 +1,75 @@ +#!groovy​ + + +node('Linux') { + +try { + String output_contents = 'output_contents' + String sources_folder = 'sources_folder' + String gh_pages_out = 'gh_pages_out' + + checkout scm + def image = null + stage('Build docker image') { + // Build the docker image using the same commit as the current 'publish.jenkins' file + image = docker.build('conan-docs', '-f .ci/Dockerfile .') // It should cache the image + } + + List branches = sh(script: 'python .ci/scripts/get_branches.py', returnStdout: true).trim().readLines() + + stage('Prepare sources as worktrees') { + String branch_argument = "" + for (branch in branches) { + branch_argument = branch_argument + " --branches=${branch}" + } + // clone sources to generate docs + sh(script: "python .ci/scripts/prepare_sources.py --sources-folder=${sources_folder} ${branch_argument}") + } + + // we have to divide the parallel blocks because if we have to generate all branches documentation + // it will fail + + def number_of_parallel_blocks = 2 + def branches_blocks = branches.collate(branches.size().intdiv(number_of_parallel_blocks)) + + for (branches_block in branches_blocks) { + Map parallelJobs = [:] + println("New block ${branches_block}") + for (branch in branches_block) { + parallelJobs[branch] = { + echo "Run parallel job for ${branch}" + image.inside { + sh(script: "python .ci/scripts/generate_documentation.py --sources-folder=${sources_folder} --branch=${branch}") + } + } + } + stage('Generate docs parallel block') { + parallelJobs.failFast = true + parallel parallelJobs + } + } + + stage('Prepare gh-pages') { + sh(script: "python .ci/scripts/prepare_gh_pages.py --sources-folder=${sources_folder} --gh-pages-folder=${gh_pages_out}") + } + + + stage('Archive generated folder') { + archiveArtifacts artifacts: "${gh_pages_out}/**/*.*" + echo "Inspect generated webpage at ${BUILD_URL}artifact/${gh_pages_out}/index.html" + } + + if (params.publish) { + stage('Publish to gh-pages') { + dir("${gh_pages_out}") { + sh 'ls' + } + } + } + } + finally { + cleanWs(cleanWhenAborted: true, cleanWhenFailure: true, cleanWhenNotBuilt: true, + cleanWhenSuccess: true, cleanWhenUnstable: true, disableDeferredWipeout: true, deleteDirs: true, + notFailBuild: true) + } +} diff --git a/.ci/scripts/common.py b/.ci/scripts/common.py new file mode 100644 index 000000000000..a45e254311c6 --- /dev/null +++ b/.ci/scripts/common.py @@ -0,0 +1,101 @@ +import os +import subprocess +from contextlib import contextmanager + + +latest_v2_folder = "2" +latest_v1_folder = "1" +latest_v1_branch = "master" + +conan_versions = { + # the first of the dictionary + # must be always the latest version + "2.0": "release/2.0", + latest_v1_folder: latest_v1_branch, + "en/1.59": "release/1.59.0", + "en/1.58": "release/1.58.0", + "en/1.57": "release/1.57.0", + "en/1.56": "release/1.56.0", + "en/1.55": "release/1.55.0", + "en/1.54": "release/1.54.0", + "en/1.53": "release/1.53.0", + "en/1.52": "release/1.52.0", + "en/1.51": "release/1.51.3", + "en/1.50": "release/1.50.2", + "en/1.49": "release/1.49.0", + "en/1.48": "release/1.48.2", + "en/1.47": "release/1.47.0", + "en/1.46": "release/1.46.2", + "en/1.45": "release/1.45.0", + "en/1.44": "release/1.44.1", + "en/1.43": "release/1.43.4", + "en/1.42": "release/1.42.2", + "en/1.41": "release/1.41.0", + "en/1.40": "release/1.40.4", + "en/1.39": "release/1.39.0", + "en/1.38": "release/1.38.0", + "en/1.37": "release/1.37.2", + "en/1.36": "release/1.36.0", + "en/1.35": "release/1.35.2", + "en/1.34": "release/1.34.1", + "en/1.33": "release/1.33.1", + "en/1.32": "release/1.32.1", + "en/1.31": "release/1.31.4", + "en/1.30": "release/1.30.2", + "en/1.29": "release/1.29.2", + "en/1.28": "release/1.28.2", + "en/1.27": "release/1.27.1", + "en/1.26": "release/1.26.1", + "en/1.25": "release/1.25.2", + "en/1.24": "release/1.24.1", + "en/1.23": "release/1.23.0", + "en/1.22": "release/1.22.3", + "en/1.21": "release/1.21.3", + "en/1.20": "release/1.20.5", + "en/1.19": "release/1.19.3", + "en/1.18": "release/1.18.5", + "en/1.17": "release/1.17.2", + "en/1.16": "release/1.16.1", + "en/1.15": "release/1.15.2", + "en/1.14": "release/1.14.5", + "en/1.13": "release/1.13.3", + "en/1.12": "release/1.12.3", + "en/1.11": "release/1.11.2", + "en/1.10": "release/1.10.2", + "en/1.9": "release/1.9.4", + "en/1.8": "release/1.8.4", + "en/1.7": "release/1.7.4", + "en/1.6": "release/1.6.1", + "en/1.5": "release/1.5.2", + "en/1.4": "release/1.4.5", + "en/1.3": "release/1.3.3", +} + +latest_v2_branch = list(conan_versions.values())[0] +latest_v2_version = list(conan_versions.keys())[0] + + +def run(cmd, capture=False): + stdout = subprocess.PIPE if capture else None + stderr = subprocess.PIPE if capture else None + process = subprocess.Popen( + cmd, stdout=stdout, stderr=stderr, shell=True) + out, err = process.communicate() + out = out.decode("utf-8") if capture else "" + err = err.decode("utf-8") if capture else "" + ret = process.returncode + output = err + out + if ret != 0: + raise Exception("Failed cmd: {}\n{}".format(cmd, output)) + return output + + +@contextmanager +def chdir(dir_path): + current = os.getcwd() + os.makedirs(dir_path, exist_ok=True) + os.chdir(dir_path) + try: + yield + finally: + os.chdir(current) diff --git a/.ci/scripts/create_redirects.py b/.ci/scripts/create_redirects.py new file mode 100644 index 000000000000..b308e0011fb2 --- /dev/null +++ b/.ci/scripts/create_redirects.py @@ -0,0 +1,45 @@ +import os +from pathlib import Path +import textwrap + + +def create_redirects(path_html, old_slug, new_slug): + """ + Redirect every html page found in sources_path from being located under the old_slug + subfolder to the new location in new_slug. For example if old_slug=en/latest and new_slug=1 + + docs.conan.io/en/latest/index.html --> redirects to --> docs.conan.io/1/index.html + """ + + path_html = Path(path_html) + + if not path_html.exists(): + print("The html directory doesn't exist") + raise SystemExit(1) + + + def replace_html_files(sources_path: Path, old_slug: str, new_slug: str): + + redirect_template = textwrap.dedent(""" + + + + + + + + """) + + html_files = sources_path.glob('**/*.html') + + for html_file in html_files: + origin = Path(old_slug) / Path(html_file).relative_to( + sources_path).parent + destination = Path(new_slug) / Path(html_file).relative_to( + sources_path) + redirect = Path(os.path.relpath(destination, origin)) + with html_file.open('w') as f: + f.write(redirect_template.format(destination=redirect)) + + + replace_html_files(path_html, old_slug, new_slug) diff --git a/.ci/scripts/generate_documentation.py b/.ci/scripts/generate_documentation.py new file mode 100644 index 000000000000..3041872aee3e --- /dev/null +++ b/.ci/scripts/generate_documentation.py @@ -0,0 +1,59 @@ +import argparse +import json +import os + +from common import chdir, conan_versions, latest_v2_folder, latest_v1_folder, latest_v2_branch, run + +parser = argparse.ArgumentParser() + +parser.add_argument("--branch", help="Docs branch to generate docs for", required=True) +parser.add_argument('--sources-folder', + help='Folder where the docs branches are cloned', required=True) +parser.add_argument('--with-pdf', default=False, action='store_true') + +args = parser.parse_args() + +branch = args.branch +with_pdf = args.with_pdf +sources_folder = args.sources_folder +output_folder = "output" + +conan_versions[latest_v2_folder] = latest_v2_branch + + +branch_folder = [k for k, v in conan_versions.items() if v == branch][0] + +print(f"branch_folder: {branch_folder}") + +with chdir(f"{sources_folder}"): + + with open(os.path.join(branch_folder, 'versions.json'), 'w') as versions_json: + json.dump(conan_versions, versions_json, indent=4) + + if branch_folder != latest_v1_folder: + run(f"rm -fr {branch_folder}/_themes/conan") + run(f"cp -a {latest_v1_folder}/_themes/. {branch_folder}/_themes/") + + # clone conan sources for autodoc + if branch_folder.startswith("2"): + # the branch in the docs for 2.0 has the same name that the one in Conan + conan_branch = branch + conan_repo_url = 'https://github.com/conan-io/conan.git' + + # clone sources + run(f"rm -rf {branch_folder}/conan_sources") + run(f"git clone --single-branch -b {conan_branch} --depth 1 {conan_repo_url} {branch_folder}/conan_sources") + + # for some reason even adding this to autodoc_mock_imports + # does not work, se we have to install the real dependency + # TODO: move this to jenkins + # run('pip3 install colorama') + + # generate html + run(f"sphinx-build -W -b html -d {branch_folder}/_build/.doctrees {branch_folder}/ {output_folder}/{branch_folder}") + + # generate pdf + if with_pdf: + run(f"sphinx-build -W -b latex -d {branch_folder}/_build/.doctrees {branch_folder}/ {branch_folder}/_build/latex") + run(f"make -C {branch_folder}/_build/latex all-pdf") + run(f"cp {branch_folder}/_build/latex/conan.pdf {output_folder}/{branch_folder}/conan.pdf") diff --git a/.ci/scripts/get_branches.py b/.ci/scripts/get_branches.py new file mode 100644 index 000000000000..5b55c30024e0 --- /dev/null +++ b/.ci/scripts/get_branches.py @@ -0,0 +1,29 @@ +import os + +from common import run, conan_versions + + +""" + Get the branches we have to build the docs for, there are two scenarios: + + 1. We changed something in the .ci scripts or change the _themes folder in the master + branch -> regenerate every branch of the docs. + + 2. If we did not touch those folders just regenerate the branch we pushed +""" +current_branch = os.getenv("BRANCH_NAME") + +current_commit = run("git rev-parse HEAD", capture=True).strip() + +previous_commit = run("git rev-parse HEAD^1", capture=True).strip() + +diff = run(f"git diff --name-only {previous_commit}..{current_commit}", capture=True) + +changed_ci = any([line.startswith(".ci") for line in diff.splitlines()]) +changed_theme = any([line.startswith("_themes") for line in diff.splitlines()]) + +if not changed_ci and not (changed_theme and current_branch == "master"): + print(current_branch) +else: + for branch in conan_versions.values(): + print(branch) diff --git a/.ci/scripts/prepare_gh_pages.py b/.ci/scripts/prepare_gh_pages.py new file mode 100644 index 000000000000..01d81c3af82f --- /dev/null +++ b/.ci/scripts/prepare_gh_pages.py @@ -0,0 +1,67 @@ +import os +import argparse +from pathlib import Path + +from common import chdir, run, latest_v1_folder, latest_v2_folder, latest_v2_version +from create_redirects import create_redirects + + +parser = argparse.ArgumentParser() + +parser.add_argument('--sources-folder', + help='Folder where the docs were created', required=True) + +parser.add_argument('--gh-pages-folder', + help='Folder to clone the gh-pages branch to', required=True) + +args = parser.parse_args() + +output_folder = os.path.join(args.sources_folder, "output") +pages_folder = args.gh_pages_folder + +with chdir(output_folder): + # FIXME: this is to not break all links from https://docs.conan.io/en/latest/ + # we copy all the /1 folder to /en/latest and then replace all html files + # there with redirects to https://docs.conan.io/en/latest/1 + # remove when most of the traffic in the docs is for 2.X docs + + # First check if we generated any docs in `latest_v1_folder` + path_latest_v1 = Path(os.path.join(latest_v1_folder)) + if path_latest_v1.exists(): + run('mkdir -p en/latest') + run(f"cp -R {latest_v1_folder}/* en/latest") + create_redirects(path_html="en/latest", old_slug="en/latest", new_slug="1") + + # 2 folder is the same as the latest 2.X, copy the generated html files to 2 folder + path_latest_v2 = Path(os.path.join(latest_v2_version)) + if path_latest_v2.exists(): + run(f"cp -R {latest_v2_version} {latest_v2_folder}") + +#run(f"rm -rf {pages_folder}") + +docs_repo_url = 'https://github.com/conan-io/docs.git' +run(f"git clone --single-branch -b gh-pages --depth 1 {docs_repo_url} {pages_folder}") + +run(f"cp -R {output_folder}/* {pages_folder}") + +run(f"cp {output_folder}/{latest_v2_folder}/404.html {pages_folder}/404.html") + +path_404 = f"{pages_folder}/404.html" + +with open(path_404, 'r') as file_404 : + contents_404 = file_404.read() + +prefix = 'https://docs.conan.io' +prefix_latest = f"{prefix}/{latest_v2_folder}" + +contents_404 = contents_404.replace('href="_', f"href=\"{prefix_latest}/_") +contents_404 = contents_404.replace('src="_', f"src=\"{prefix_latest}/_") +contents_404 = contents_404.replace('alt="_', f"alt=\"{prefix_latest}/_") +contents_404 = contents_404.replace('internal" href="', f"internal\" href=\"{prefix_latest}/") +contents_404 = contents_404.replace('"search.html"', f"\"{prefix_latest}/search.html\"") +contents_404 = contents_404.replace('"genindex.html"', f"\"{prefix_latest}/genindex.html\"") + +with open(path_404, 'w') as file: + file.write(contents_404) + +# gh-pages prepared to push diff --git a/.ci/scripts/prepare_sources.py b/.ci/scripts/prepare_sources.py new file mode 100644 index 000000000000..b7382a859604 --- /dev/null +++ b/.ci/scripts/prepare_sources.py @@ -0,0 +1,35 @@ +import os +import argparse +from pathlib import Path + +from common import chdir, conan_versions, latest_v1_branch, run + +parser = argparse.ArgumentParser() + +parser.add_argument('--sources-folder', + help='Folder where the docs branches are cloned', required=True) + +parser.add_argument('--branches', action='append', + help='List of branches (separated by a space) to generate docs for.\ + If not specified it will re-generate all branches in docs repo.', required=True) + +args = parser.parse_args() + +sources_folder = args.sources_folder +branches = args.branches + +# we have to clone master always as it has the templates and styles for the whole docs +if latest_v1_branch not in branches: + branches.append(latest_v1_branch) + +print(f"Prepare docs for: {branches}") + +# Prepare sources as worktrees +if not Path(f"{sources_folder}/tmp").is_dir(): + run(f"git clone --bare https://github.com/conan-io/docs.git {sources_folder}/tmp") + +with chdir(f"{sources_folder}/tmp"): + for folder, branch in conan_versions.items(): + if branch in branches and not Path(f"../{folder}").is_dir(): + run(f"git fetch origin {branch}:{branch}") + run(f"git worktree add ../{folder} {branch}") diff --git a/create_redirects.py b/create_redirects.py deleted file mode 100644 index 24507c027964..000000000000 --- a/create_redirects.py +++ /dev/null @@ -1,56 +0,0 @@ -import argparse -import os -from pathlib import Path -import textwrap - -parser = argparse.ArgumentParser() - -parser.add_argument("--old", help="old slug like for example /en/latest") -parser.add_argument("--new", help="new slug we want to redirect to like /1") -parser.add_argument("path_html", - help="path where the generated html files are") - -args = parser.parse_args() - -old_slug = args.old -new_slug = args.new -path_html = Path(args.path_html) - -if not path_html.exists(): - print("The html directory doesn't exist") - raise SystemExit(1) - - -""" -Redirect every html page found in sources_path from being located under the old_slug -subfolder to the new location in new_slug. For example if old_slug=en/latest and new_slug=1 - -docs.conan.io/en/latest/index.html --> redirects to --> docs.conan.io/1/index.html -""" - - -def replace_html_files(sources_path: Path, old_slug: str, new_slug: str): - - redirect_template = textwrap.dedent(""" - - - - - - - - """) - - html_files = sources_path.glob('**/*.html') - - for html_file in html_files: - origin = Path(old_slug) / Path(html_file).relative_to( - sources_path).parent - destination = Path(new_slug) / Path(html_file).relative_to( - sources_path) - redirect = Path(os.path.relpath(destination, origin)) - with html_file.open('w') as f: - f.write(redirect_template.format(destination=redirect)) - - -replace_html_files(path_html, old_slug, new_slug) diff --git a/requirements.txt b/requirements.txt index 5e98bd540b65..7a20c8a33d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ sphinx-sitemap>=2.1.0 sphinxcontrib-spelling sphinx-notfound-page jinja2<=3.0.3 +colorama>=0.4.3, <0.5.0