From 47f59f82017101ac0ff2d810b688e59f7bba16be Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Sat, 11 Jul 2026 12:24:48 -0400 Subject: [PATCH] Revert "Add workflow to automatically generate and commit site HTML (#697)" This reverts commit 59719624d8f4bb548e1042654e60cb787b2824cf. --- .github/CONTRIBUTING.md | 6 +-- .github/PULL_REQUEST_TEMPLATE.md | 1 + .github/actions/build-html/action.yml | 21 -------- .github/workflows/doc_gen.yml | 76 +++++++++++++++++++++++++++ .github/workflows/html-build.yml | 16 ------ .github/workflows/html-push.yml | 43 --------------- .gitignore | 2 +- README.md | 28 ++++++---- _config.yml | 2 +- content | 1 + site/sitemap.xml | 39 +++++++------- sitemap.xml | 15 ++---- 12 files changed, 123 insertions(+), 127 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/actions/build-html/action.yml create mode 100644 .github/workflows/doc_gen.yml delete mode 100644 .github/workflows/html-build.yml delete mode 100644 .github/workflows/html-push.yml create mode 120000 content diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ab2057b8e76..7b5dc0b3487 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,5 +1 @@ -# Contributing to this repo - -Build and test your changes locally according to the instructions in [README](../README.md). - -Once you've done that, submit a pull request with your changes. You only need to commit your changes to the Markdown source. A GitHub Actions workflow will [generate the corresponding HTML and push it for you](./workflows/html-push.yml). +Make sure that you generate site HTML with `jekyll build`, and include the changes to the HTML in your pull request also. See README.md for more information. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..9e1ffb348a5 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1 @@ + diff --git a/.github/actions/build-html/action.yml b/.github/actions/build-html/action.yml deleted file mode 100644 index 76f7767416a..00000000000 --- a/.github/actions/build-html/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Build HTML - -description: Set up Ruby and run the Jekyll build. - -inputs: - ruby-version: - description: Ruby version to use. - required: false - default: "3.4" - -runs: - using: composite - steps: - - name: Set up Ruby and Bundler - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ inputs.ruby-version }} - bundler-cache: true - - name: Run documentation build - shell: bash - run: bundle exec jekyll build diff --git a/.github/workflows/doc_gen.yml b/.github/workflows/doc_gen.yml new file mode 100644 index 00000000000..01478f92735 --- /dev/null +++ b/.github/workflows/doc_gen.yml @@ -0,0 +1,76 @@ +name: Check document generation + +on: + push: + branches: + - asf-site + pull_request: + branches: + - asf-site + +jobs: + lint: + name: check whether all documentation was generated with the right Jekyll version + runs-on: ubuntu-24.04 + steps: + - name: Free up disk space + shell: 'script -q -e -c "bash {0}"' + run: | + echo "==================================" + echo "Free up disk space on CI system" + echo "==================================" + + echo "Listing top 100 largest packages (from large to small)" + printf "Installed-Size\tPackage\n" + dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n -r | head -n 100 + df -h + + echo "Removing large packages" + rm -rf /__t/CodeQL + rm -rf /__t/go + rm -rf /__t/node + + apt-get remove --purge -y '^aspnet.*' || true + apt-get remove --purge -y '^dotnet-.*' || true + apt-get remove --purge -y '^llvm-.*' || true + apt-get remove --purge -y 'php.*' || true + apt-get remove --purge -y '^mongodb-.*' || true + apt-get remove --purge -y 'gfortran-11' || true + apt-get remove --purge -y 'humanity-icon-theme' || true + apt-get remove --purge -y 'nodejs-doc' || true + apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox || true + apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri || true + apt-get autoremove --purge -y + apt-get clean + + df -h + - name: Checkout Spark Website repository + uses: actions/checkout@v2 + - name: Install dependencies for documentation generation + run: | + sudo apt-get update -y + sudo apt-get install -y ruby ruby-dev + sudo gem install bundler --version 2.4.19 + bundle install + - name: Run documentation build + run: | + export LC_ALL=C.UTF-8 + export LANG=C.UTF-8 + OLD_IFS=$IFS + IFS= + GEN_ERRORS=$(bundle exec jekyll build 3>&2 2>&1 1>&3) + if [ $(echo $GEN_ERRORS| grep -v -e '^$'| grep -c -v "rubygems_integration") -ne 0 ]; then + echo "Error during document generation:" + echo $GEN_ERRORS + exit 1 + fi + IFS=$OLD_IFS + CHANGED_FILE=( $(git ls-files --modified --other --exclude-standard --directory | grep -v sitemap.xml | grep -v llms.txt) ) + if [ ${#CHANGED_FILE[@]} -ne 0 ]; then + echo "Not all documentation was generated and/or not the right Jekyll version was used! Modified / untracked files (excluding sitemap.xml):" + echo ${CHANGED_FILE[*]} + echo "Git diff (excluding sitemap.xml):" + git diff -- . ':(exclude)site/sitemap.xml' + exit 1 + fi + shell: /bin/bash {0} diff --git a/.github/workflows/html-build.yml b/.github/workflows/html-build.yml deleted file mode 100644 index b94dfebe2b3..00000000000 --- a/.github/workflows/html-build.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Build HTML - -on: - pull_request: - branches: - - asf-site - -jobs: - build: - name: Build HTML - runs-on: ubuntu-24.04 - steps: - - name: Checkout Spark Website repository - uses: actions/checkout@v7 - - name: Build HTML - uses: ./.github/actions/build-html diff --git a/.github/workflows/html-push.yml b/.github/workflows/html-push.yml deleted file mode 100644 index 15160e7b4cd..00000000000 --- a/.github/workflows/html-push.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build and Push HTML - -on: - push: - branches: - - asf-site - -jobs: - commit: - name: Build and commit HTML to `asf-site` - # This condition is important. We don't want to trigger this job if the last - # commit was created _by_ this job! - if: >- - !( - contains(github.event.head_commit.message, '[html]') && - github.event.head_commit.author.name == 'github-actions[bot]' - ) - # Not technically necessary, but helps avoid spurious failures if multiple - # commits are pushed in rapid succession. - concurrency: - group: html-push-${{ github.ref }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - permissions: - contents: write - steps: - - name: Checkout Spark Website repository - uses: actions/checkout@v7 - - name: Build HTML - uses: ./.github/actions/build-html - - name: Commit and push generated HTML - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - # `-f` because we told git to otherwise ignore `site/` - git add -f site/ - if git diff --cached --quiet; then - echo "No changes to commit." - else - COMMIT_TITLE=$(git log -1 --pretty=%s) - git commit -m "[html] $COMMIT_TITLE" - git push - fi diff --git a/.gitignore b/.gitignore index 90ef9e92051..d0d5f1c2673 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ target/ .jekyll-cache/ .jekyll-metadata .local_ruby_bundle -site/ +site/python diff --git a/README.md b/README.md index 7b776039361..bef477a201f 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,4 @@ -# Apache Spark Main Website - -This repository captures the main Apache Spark website located at https://spark.apache.org. The programming docs are [in the main Spark repo][1], not here. - -[1]: https://github.com/apache/spark/tree/master/docs - -To contribute changes, see [CONTRIBUTING](.github/CONTRIBUTING.md). - -## Generating the website HTML locally +## Generating the website HTML In this directory you will find text files formatted using Markdown, with an `.md` suffix. @@ -36,6 +28,22 @@ of Spark from the Spark source repository and then copied to the website under t directory. See the instructions for building those in the readme in the Spark project's `/docs` directory. +## Rouge and Pygments + +We also use [Rouge](https://github.com/rouge-ruby/rouge) for syntax highlighting in documentation Markdown pages. +Its HTML output is compatible with CSS files designed for [Pygments](https://pygments.org/). + +To mark a block of code in your Markdown to be syntax highlighted by `jekyll` during the +compile phase, use the following syntax: + + {% highlight scala %} + // Your Scala code goes here, you can replace Scala with many other + // supported languages too. + {% endhighlight %} + +You probably don't need to install that unless you want to regenerate the Pygments CSS file. +It requires Python, and can be installed by running `sudo easy_install Pygments`. + ## Merge PR -To merge a pull request, use the `merge_pr.py` script. This script also squashes the commits. +To merge pull request, use the `merge_pr.py` script which also squashes the commits. diff --git a/_config.yml b/_config.yml index 5e5be4c0c88..98f05068c08 100644 --- a/_config.yml +++ b/_config.yml @@ -4,6 +4,6 @@ kramdown: entity_output: symbol permalink: none destination: site -exclude: ['README.md', 'LICENSE', 'merge_pr.py', 'Gemfile', 'Gemfile.lock'] +exclude: ['README.md', 'content', 'LICENSE', 'merge_pr.py', 'Gemfile', 'Gemfile.lock'] keep_files: ['docs', 'static', 'llms.txt'] url: https://spark.apache.org diff --git a/content b/content new file mode 120000 index 00000000000..d97e1006371 --- /dev/null +++ b/content @@ -0,0 +1 @@ +site \ No newline at end of file diff --git a/site/sitemap.xml b/site/sitemap.xml index 19f2ddf2f1d..3ef7e411a11 100644 --- a/site/sitemap.xml +++ b/site/sitemap.xml @@ -1163,10 +1163,7 @@ https://spark.apache.org/releases/spark-release-0-3.html weekly - - https://spark.apache.org/ - weekly - + https://spark.apache.org/404.html weekly @@ -1208,19 +1205,23 @@ weekly - https://spark.apache.org/graphx/ + https://spark.apache.org/history.html weekly - https://spark.apache.org/history.html + https://spark.apache.org/improvement-proposals.html weekly - https://spark.apache.org/improvement-proposals.html + https://spark.apache.org/spark-connect/ weekly - https://spark.apache.org/mailing-lists.html + https://spark.apache.org/pandas-on-spark/ + weekly + + + https://spark.apache.org/graphx/ weekly @@ -1228,45 +1229,46 @@ weekly - https://spark.apache.org/news/ + https://spark.apache.org/streaming/ weekly - https://spark.apache.org/pandas-on-spark/ + https://spark.apache.org/news/ weekly - https://spark.apache.org/powered-by.html + https://spark.apache.org/screencasts/ weekly - https://spark.apache.org/release-process.html + https://spark.apache.org/sql/ weekly - https://spark.apache.org/research.html + https://spark.apache.org/ weekly - https://spark.apache.org/screencasts/ + https://spark.apache.org/mailing-lists.html weekly - https://spark.apache.org/security.html + https://spark.apache.org/powered-by.html weekly - https://spark.apache.org/spark-connect/ + https://spark.apache.org/release-process.html weekly - https://spark.apache.org/sql/ + https://spark.apache.org/research.html weekly - https://spark.apache.org/streaming/ + https://spark.apache.org/security.html weekly + https://spark.apache.org/third-party-projects.html weekly @@ -1279,4 +1281,5 @@ https://spark.apache.org/versioning-policy.html weekly + diff --git a/sitemap.xml b/sitemap.xml index 71fb8b093dc..257359ef91c 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -151,18 +151,9 @@ sitemap: false weekly {% endfor %} -{%- comment -%} -Explicitly sort `site.pages` so that the order is consistent and we don't get spurious git diffs. -`site.posts` doesn't have this issue because it's already sorted. -See: https://jekyllrb.com/docs/variables/#site-variables -{%- endcomment -%} -{%- assign sorted_pages = site.pages | sort: "url" -%} -{%- for page in sorted_pages -%} -{%- if page.sitemap != false -%} - +{% for page in site.pages %}{% if page.sitemap != false %} {{ site.url }}{{ page.url }} weekly - -{% endif %} -{%- endfor -%} +{% endif %} +{% endfor %}