Fix: update htmlproofer to ignore new files as broken links (#343)

* Fix: update htmlproofer

* Fix:main branch needed

* Add new page

* Fix: remove bad link but it still has a new page

* Fix: speed up with typheus

* Fix: ssl 1

* Fix: print out errors

* Fix: print out errors

* Update _pages/home.md

* Update build-site.yml

* Removing caching

* what happened to the key, buddy?

Author: lwasser

.github/workflows/build-site.yml

@@ -6,13 +6,22 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  github-pages:
+  build-site:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the code
         uses: actions/checkout@v4
 
+      - name: Fetch main
+        run: |
+          git fetch origin main --depth 1
+          git branch
+
       # Use GitHub Actions' cache to shorten build times and decrease load on servers
       - name: Use cache to shorten build time
         uses: actions/cache@v4
@@ -28,11 +37,26 @@ jobs:
           build_only: true
           build_dir: _site
 
-      - name: Check HTML using htmlproofer
-        uses: chabad360/htmlproofer@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
         with:
-          directory: "_site"
-          arguments: |
-            --ignore-urls "https://fonts.googleapis.com,https://fonts.gstatic.com,_site/_posts/README/index.html"
-            --ignore-files "/.+\/_posts\/README.md"
-            --ignore-status-codes "0, 200, 403, 429, 503, 999"
+          ruby-version: 3
+      - name: Install gems
+        run: |
+          gem install html-proofer
+          gem install typhoeus
+
+      - name: Cache HTMLProofer
+        id: cache-htmlproofer
+        uses: actions/cache@v4
+        with:
+          path: tmp/.htmlproofer
+          key: ${{ runner.os }}-htmlproofer
+
+      - name: Run htmlproofer
+        run: |
+          ruby scripts/htmlproofer.rb
+          ls
+
+      - name: Error log
+        run: cat html_proofer_errors.txt

.gitignore

@@ -9,3 +9,4 @@ images/full-images/
 all_contribs.pickle
 .vale.ini
 styles/*
+tmp/*

_pages/home.md

@@ -40,6 +40,7 @@ peer-review:
     btn_class: btn--inverse
 ---
 
+
 {% include feature_row id="mission" type="center" %}
 
 ## Peer review of Python software to support open science

_posts/2024-01-03-testme.md

@@ -0,0 +1,17 @@
+---
+layout: single
+title: "CZI Funds pyOpenSci"
+excerpt: "CZI has received funding from the Chan Zuckerberg Foundation. Learn more about how this funding will support pyOpenSci over the next 2 years."
+author: "Leah Wasser"
+permalink: /blog/czi-funds-pyOpenSci-2024.html
+header:
+  overlay_color: "#666"
+  overlay_filter: 0.6
+categories:
+  - blog-post
+  - community
+toc: true
+comments: true
+---
+
+i am a new page with a no bad links

scripts/htmlproofer.rb

@@ -0,0 +1,49 @@
+require "html-proofer"
+require "typhoeus"
+
+begin
+  directories = ['../_site']
+  status_codes = [0, 200, 301, 403, 429, 503, 999]
+  merge_base = %x(git merge-base origin/main HEAD).chomp
+  diffable_files = %x(git diff -z --name-only --diff-filter=AC #{merge_base}).split("\0")
+  diffable_files = diffable_files.select do |filename|
+    next true if directories.include?(File.dirname(filename))
+
+    filename.end_with?(".md")
+  end.map { |f| Regexp.new(File.basename(f, File.extname(f))) }
+
+  # Add urls for it to ignore
+  diffable_files << %r{^https://fonts\.gstatic\.com.*}
+  diffable_files << %r{^https://fonts\.googleapis\.com.*}
+
+  # Create a file to capture errors
+  errors = StringIO.new
+  HTMLProofer.check_directory(
+    "./_site",
+    {
+      ignore_urls: diffable_files,
+      ignore_status_codes: status_codes,
+      cache: {
+        timeframe: {
+          external: "7d",
+          internal: "7d"
+        }
+      },
+      typhoeus: {
+        verbose: false,
+        ssl_verifyhost: 1
+      }
+    }
+  ).run { |message| errors.puts message }
+
+   # Write errors to a text file
+   File.write("html_proofer_errors.txt", errors.string)
+
+rescue StandardError => e
+  # Print out the error message
+  puts "An error occurred: #{e.message}"
+  # Print out backtrace if needed
+  puts e.backtrace.join("\n")
+  # Exit with a non-zero status code to indicate failure
+  exit 1
+end