Update Packages #741
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Packages | |
| on: | |
| schedule: | |
| # Run hourly at the top of each hour | |
| - cron: '0 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| update-packages: | |
| name: Update packages and index | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # Needed to push changes | |
| steps: | |
| - name: Debug workflow trigger | |
| run: | | |
| echo "=== Update Workflow Debug Information ===" | |
| echo "Event name: ${{ github.event_name }}" | |
| echo "Repository: ${{ github.repository }}" | |
| echo "Ref: ${{ github.ref }}" | |
| echo "SHA: ${{ github.sha }}" | |
| echo "Actor: ${{ github.actor }}" | |
| echo "Workflow: ${{ github.workflow }}" | |
| echo "Job: ${{ github.job }}" | |
| echo "Run ID: ${{ github.run_id }}" | |
| echo "Run number: ${{ github.run_number }}" | |
| echo "Current time: $(date)" | |
| echo "=============================================" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN || github.token }} | |
| fetch-depth: 0 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Install Playwright browsers | |
| run: bunx playwright install chromium --with-deps | |
| - name: Show initial git status | |
| run: | | |
| echo "=== Initial Git Status ===" | |
| git status --porcelain | |
| echo "=== End Initial Git Status ===" | |
| - name: Fetch all packages | |
| id: fetch-packages | |
| run: | | |
| echo "Starting package fetch..." | |
| # Create a temporary file to capture the JSON output safely | |
| TEMP_JSON_FILE=$(mktemp) | |
| # First run the fetch command normally to do the actual work | |
| echo "Running fetch command to update packages..." | |
| if timeout 3600 bun bin/cli.ts fetch --all --verbose --concurrency 8 --timeout 15000; then | |
| echo "Fetch command completed successfully" | |
| FETCH_SUCCESS=true | |
| else | |
| FETCH_EXIT_CODE=$? | |
| echo "Fetch command failed with exit code: $FETCH_EXIT_CODE" | |
| FETCH_SUCCESS=false | |
| fi | |
| # Now determine what was actually updated by checking git status | |
| echo "Checking git status to determine updated packages..." | |
| CHANGED_FILES=$(git status --porcelain src/packages/ | grep "^ M " | awk '{print $2}' | sed 's|src/packages/||' | sed 's|\.ts$||' | head -20) | |
| if [ -n "$CHANGED_FILES" ]; then | |
| echo "Found changed package files:" | |
| echo "$CHANGED_FILES" | |
| # Convert to JSON array | |
| UPDATED_PACKAGES_JSON="[" | |
| FIRST=true | |
| TOTAL_UPDATED=0 | |
| while IFS= read -r file; do | |
| if [ -n "$file" ]; then | |
| if [ "$FIRST" = true ]; then | |
| FIRST=false | |
| else | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON," | |
| fi | |
| # Escape package name for JSON | |
| ESC_PKG=$(echo "$file" | sed 's/"/\\"/g') | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON\"$ESC_PKG\"" | |
| TOTAL_UPDATED=$((TOTAL_UPDATED + 1)) | |
| fi | |
| done <<< "$CHANGED_FILES" | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON]" | |
| else | |
| echo "No package files were changed" | |
| UPDATED_PACKAGES_JSON="[]" | |
| TOTAL_UPDATED=0 | |
| fi | |
| # Create JSON result based on fetch success and git changes | |
| if [ "$FETCH_SUCCESS" = true ]; then | |
| printf '{"success":true,"totalUpdated":%d,"totalProcessed":530,"updatedPackages":%s}' \ | |
| "$TOTAL_UPDATED" "$UPDATED_PACKAGES_JSON" > "$TEMP_JSON_FILE" | |
| else | |
| printf '{"success":false,"error":"fetch command failed","totalUpdated":0,"totalProcessed":0,"updatedPackages":[]}' > "$TEMP_JSON_FILE" | |
| fi | |
| # Check file size and truncate if necessary to prevent GitHub Actions output limits | |
| FILE_SIZE=$(wc -c < "$TEMP_JSON_FILE" 2>/dev/null || echo "0") | |
| echo "JSON output size: $FILE_SIZE bytes" | |
| if [ "$FILE_SIZE" -gt 262144 ]; then # 256KB limit | |
| echo "JSON output too large ($FILE_SIZE bytes), creating summary instead" | |
| # Extract just the essential info for commit message generation | |
| # First verify the JSON is valid before processing | |
| if ! jq empty "$TEMP_JSON_FILE" 2>/dev/null; then | |
| echo "JSON is invalid during size check, creating minimal fallback" | |
| echo '{"success":false,"error":"invalid_json_during_truncation","totalUpdated":0,"totalProcessed":0,"updatedPackages":[]}' > "$TEMP_JSON_FILE" | |
| fi | |
| SUCCESS=$(jq -r '.success // false' "$TEMP_JSON_FILE" 2>/dev/null || echo "false") | |
| TOTAL_UPDATED=$(jq -r '.totalUpdated // 0' "$TEMP_JSON_FILE" 2>/dev/null || echo "0") | |
| TOTAL_PROCESSED=$(jq -r '.totalProcessed // 0' "$TEMP_JSON_FILE" 2>/dev/null || echo "0") | |
| # Get first 15 updated packages for commit message, handling both possible array names | |
| # Create a simple array of package names, avoiding complex jq operations that can cause broken pipes | |
| UPDATED_PACKAGES_TEMP=$(mktemp) | |
| { | |
| jq -r '.updatedPackagesMixed[]? // empty' "$TEMP_JSON_FILE" 2>/dev/null || true | |
| jq -r '.updatedPackages[]? // empty' "$TEMP_JSON_FILE" 2>/dev/null || true | |
| } 2>/dev/null | head -15 > "$UPDATED_PACKAGES_TEMP" || echo "" > "$UPDATED_PACKAGES_TEMP" | |
| # Create JSON array from the package names | |
| UPDATED_PACKAGES_JSON="[" | |
| FIRST=true | |
| if [ -f "$UPDATED_PACKAGES_TEMP" ]; then | |
| while IFS= read -r pkg; do | |
| if [ -n "$pkg" ] && [ "$pkg" != "" ]; then | |
| if [ "$FIRST" = true ]; then | |
| FIRST=false | |
| else | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON," | |
| fi | |
| # Escape package name for JSON and sanitize | |
| ESC_PKG=$(echo "$pkg" | sed 's/"/\\"/g' | tr -d '\0\n\r') | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON\"$ESC_PKG\"" | |
| fi | |
| done < "$UPDATED_PACKAGES_TEMP" | |
| fi | |
| UPDATED_PACKAGES_JSON="$UPDATED_PACKAGES_JSON]" | |
| rm -f "$UPDATED_PACKAGES_TEMP" | |
| # Create a smaller summary JSON with consistent structure using printf | |
| printf '{"success":%s,"totalUpdated":%s,"totalProcessed":%s,"updatedPackages":%s,"note":"truncated_for_size"}' \ | |
| "$SUCCESS" "$TOTAL_UPDATED" "$TOTAL_PROCESSED" "$UPDATED_PACKAGES_JSON" > "$TEMP_JSON_FILE" | |
| # Validate the created summary JSON | |
| if ! jq empty "$TEMP_JSON_FILE" 2>/dev/null; then | |
| echo "Created summary JSON is invalid, using minimal fallback" | |
| echo '{"success":false,"error":"summary_json_invalid","totalUpdated":0,"totalProcessed":0,"updatedPackages":[],"note":"fallback_due_to_invalid_summary"}' > "$TEMP_JSON_FILE" | |
| fi | |
| echo "Created summary JSON ($(wc -c < "$TEMP_JSON_FILE" 2>/dev/null || echo "0") bytes)" | |
| fi | |
| # Validate JSON before outputting | |
| if ! jq empty "$TEMP_JSON_FILE" 2>/dev/null; then | |
| echo "Invalid JSON detected, creating fallback" | |
| echo '{"success":false,"error":"invalid_json","totalUpdated":0,"totalProcessed":0,"updatedPackages":[]}' > "$TEMP_JSON_FILE" | |
| fi | |
| # Debug: Show final JSON structure | |
| echo "=== FETCH STEP DEBUG ===" | |
| echo "Final JSON structure:" | |
| cat "$TEMP_JSON_FILE" | |
| echo "" | |
| echo "JSON file size: $(wc -c < "$TEMP_JSON_FILE" 2>/dev/null || echo "0") bytes" | |
| echo "JSON validation result: $(jq empty "$TEMP_JSON_FILE" 2>/dev/null && echo "VALID" || echo "INVALID")" | |
| echo "Success field: $(jq -r '.success // "not_found"' "$TEMP_JSON_FILE" 2>/dev/null || echo "jq_failed")" | |
| echo "Total updated: $(jq -r '.totalUpdated // "not_found"' "$TEMP_JSON_FILE" 2>/dev/null || echo "jq_failed")" | |
| echo "=== END FETCH DEBUG ===" | |
| # Use the file content for GitHub output - single line approach | |
| # Convert JSON to single line and escape it properly for GitHub Actions | |
| JSON_LINE=$(cat "$TEMP_JSON_FILE" | jq -c . 2>/dev/null || cat "$TEMP_JSON_FILE") | |
| echo "fetch_result=${JSON_LINE}" >> $GITHUB_OUTPUT | |
| # Clean up temp file | |
| rm -f "$TEMP_JSON_FILE" | |
| timeout-minutes: 65 | |
| - name: Show git status after fetch | |
| run: | | |
| echo "=== Git Status After Fetch ===" | |
| git status --porcelain | |
| echo "=== End Git Status After Fetch ===" | |
| - name: Generate index | |
| run: bun bin/cli.ts generate-index | |
| timeout-minutes: 5 | |
| - name: Generate aliases | |
| run: bun bin/cli.ts generate-aliases | |
| timeout-minutes: 5 | |
| - name: Generate documentation | |
| run: bun bin/cli.ts generate-docs | |
| timeout-minutes: 5 | |
| - name: Show git status after all generation | |
| run: | | |
| echo "=== Git Status After All Generation ===" | |
| git status --porcelain | |
| echo "=== End Git Status After All Generation ===" | |
| - name: Generate commit message | |
| id: commit-message | |
| run: | | |
| # Process the fetch result directly from the GitHub output variable | |
| # to avoid broken pipe issues with large JSON content | |
| echo "Processing fetch result for commit message generation..." | |
| # Create a temporary file to safely handle the JSON data | |
| FETCH_JSON_FILE=$(mktemp) | |
| # Write the fetch result to the temp file, handling potential issues | |
| FETCH_RESULT='${{ steps.fetch-packages.outputs.fetch_result }}' | |
| # Debug: Check if fetch result exists and show its size | |
| echo "=== COMMIT MESSAGE DEBUG ===" | |
| echo "Fetch result variable length: ${#FETCH_RESULT}" | |
| echo "First 200 chars of fetch result: $(echo "$FETCH_RESULT" | head -c 200 2>/dev/null || echo "empty")" | |
| echo "Fetch result ends with: $(echo "$FETCH_RESULT" | tail -c 50 2>/dev/null || echo "empty")" | |
| if [ -z "$FETCH_RESULT" ] || [ "$FETCH_RESULT" = "" ]; then | |
| echo "No fetch result found, using fallback values" | |
| echo '{"success":false,"error":"no_fetch_result","totalUpdated":0,"totalProcessed":0,"updatedPackages":[]}' > "$FETCH_JSON_FILE" | |
| else | |
| echo "$FETCH_RESULT" > "$FETCH_JSON_FILE" | |
| # Validate the JSON first | |
| if ! jq empty "$FETCH_JSON_FILE" 2>/dev/null; then | |
| echo "Invalid JSON in fetch result, using fallback values" | |
| echo '{"success":false,"error":"invalid_json","totalUpdated":0,"totalProcessed":0,"updatedPackages":[]}' > "$FETCH_JSON_FILE" | |
| else | |
| echo "Successfully loaded fetch result JSON" | |
| fi | |
| fi | |
| # Debug: Show what we actually wrote to the temp file | |
| echo "Temp file size: $(wc -c < "$FETCH_JSON_FILE" 2>/dev/null || echo "0") bytes" | |
| echo "Temp file content (first 200 chars): $(head -c 200 "$FETCH_JSON_FILE" 2>/dev/null || echo "empty")" | |
| echo "=== END COMMIT MESSAGE DEBUG ===" | |
| # Extract key values directly from the file using jq with error handling | |
| FETCH_SUCCESS=$(jq -r '.success // false' "$FETCH_JSON_FILE" 2>/dev/null || echo "false") | |
| if [ "$FETCH_SUCCESS" != "true" ]; then | |
| echo "Fetch was not successful (success=$FETCH_SUCCESS), skipping commit" | |
| ERROR_MSG=$(jq -r '.error // "unknown_error"' "$FETCH_JSON_FILE" 2>/dev/null || echo "unknown_error") | |
| echo "Fetch error: $ERROR_MSG" | |
| echo "commit_message=chore: fetch failed ($ERROR_MSG)" >> $GITHUB_OUTPUT | |
| echo "total_updated=0" >> $GITHUB_OUTPUT | |
| echo "should_commit=false" >> $GITHUB_OUTPUT | |
| rm -f "$FETCH_JSON_FILE" | |
| exit 0 | |
| fi | |
| # Extract package counts safely | |
| TOTAL_UPDATED=$(jq -r '.totalUpdated // 0' "$FETCH_JSON_FILE" 2>/dev/null || echo "0") | |
| TOTAL_PROCESSED=$(jq -r '.totalProcessed // 0' "$FETCH_JSON_FILE" 2>/dev/null || echo "0") | |
| # Extract first few package names for commit message | |
| # Use a more robust approach that handles large arrays without broken pipes | |
| UPDATED_PACKAGES=$(jq -r ' | |
| if .updatedPackages then .updatedPackages[] | |
| elif .updatedPackagesMixed then .updatedPackagesMixed[] | |
| else empty | |
| end' "$FETCH_JSON_FILE" 2>/dev/null | head -15 | sed 's/[^a-zA-Z0-9.-]/_/g' | tr '\n' ' ' | tr -d '\0' | xargs 2>/dev/null || echo "") | |
| # Debug output with safe string operations | |
| echo "Total packages processed: $TOTAL_PROCESSED" | |
| echo "Total packages updated: $TOTAL_UPDATED" | |
| echo "Sample packages: $(echo "$UPDATED_PACKAGES" | cut -d' ' -f1-3 | tr -d '\n')" | |
| # Generate commit message based on update count | |
| if [ "$TOTAL_UPDATED" -eq 0 ]; then | |
| COMMIT_MSG="chore: update packages (no changes detected)" | |
| SHOULD_COMMIT="false" | |
| elif [ "$TOTAL_UPDATED" -le 4 ]; then | |
| # For small numbers, list all packages | |
| PACKAGE_LIST=$(echo "$UPDATED_PACKAGES" | cut -d' ' -f1-$TOTAL_UPDATED | tr ' ' ',' | sed 's/,/, /g' | sed 's/, $//' | tr -d '\n\r') | |
| if [ -n "$PACKAGE_LIST" ] && [ "$PACKAGE_LIST" != "" ] && [ "$PACKAGE_LIST" != " " ]; then | |
| COMMIT_MSG="chore: update $PACKAGE_LIST" | |
| else | |
| COMMIT_MSG="chore: update $TOTAL_UPDATED packages" | |
| fi | |
| SHOULD_COMMIT="true" | |
| else | |
| # For larger numbers, calculate how many we can fit in ~72 chars total | |
| # "chore: update " = 14 chars, " and X other deps" = ~15 chars, so ~43 chars for package names | |
| COMMIT_PREFIX="chore: update " | |
| MAX_PACKAGE_CHARS=43 | |
| # Build package list up to the character limit | |
| PACKAGE_LIST="" | |
| PACKAGE_COUNT=0 | |
| CURRENT_LENGTH=0 | |
| for pkg in $UPDATED_PACKAGES; do | |
| if [ -z "$pkg" ] || [ "$pkg" = "" ]; then | |
| continue | |
| fi | |
| # Calculate length if we add this package | |
| if [ -z "$PACKAGE_LIST" ]; then | |
| NEW_LENGTH=${#pkg} | |
| else | |
| NEW_LENGTH=$((CURRENT_LENGTH + 2 + ${#pkg})) # +2 for ", " | |
| fi | |
| # Stop if adding this package would exceed our limit | |
| if [ $NEW_LENGTH -gt $MAX_PACKAGE_CHARS ]; then | |
| break | |
| fi | |
| # Add the package | |
| if [ -z "$PACKAGE_LIST" ]; then | |
| PACKAGE_LIST="$pkg" | |
| else | |
| PACKAGE_LIST="$PACKAGE_LIST, $pkg" | |
| fi | |
| CURRENT_LENGTH=$NEW_LENGTH | |
| PACKAGE_COUNT=$((PACKAGE_COUNT + 1)) | |
| done | |
| # Calculate remaining packages | |
| REMAINING=$((TOTAL_UPDATED - PACKAGE_COUNT)) | |
| if [ -n "$PACKAGE_LIST" ] && [ "$PACKAGE_LIST" != "" ] && [ "$REMAINING" -gt 0 ]; then | |
| if [ "$REMAINING" -eq 1 ]; then | |
| COMMIT_MSG="chore: update $PACKAGE_LIST and 1 other dep" | |
| else | |
| COMMIT_MSG="chore: update $PACKAGE_LIST and $REMAINING other deps" | |
| fi | |
| elif [ -n "$PACKAGE_LIST" ] && [ "$PACKAGE_LIST" != "" ]; then | |
| # All packages fit, no "and X other" needed | |
| COMMIT_MSG="chore: update $PACKAGE_LIST" | |
| else | |
| # Fallback if package list is empty | |
| COMMIT_MSG="chore: update $TOTAL_UPDATED packages" | |
| fi | |
| SHOULD_COMMIT="true" | |
| fi | |
| # Sanitize commit message and ensure it's not too long (GitHub limit is 72 chars for subject) | |
| COMMIT_MSG=$(echo "$COMMIT_MSG" | tr -d '\n\r\0' | sed 's/[[:cntrl:]]//g' | cut -c1-72) | |
| # Set outputs | |
| echo "commit_message=$COMMIT_MSG" >> $GITHUB_OUTPUT | |
| echo "total_updated=$TOTAL_UPDATED" >> $GITHUB_OUTPUT | |
| echo "should_commit=$SHOULD_COMMIT" >> $GITHUB_OUTPUT | |
| # Final debug output | |
| echo "Generated commit message: $COMMIT_MSG" | |
| echo "Should commit: $SHOULD_COMMIT" | |
| echo "Total updated: $TOTAL_UPDATED" | |
| # Clean up temp file | |
| rm -f "$FETCH_JSON_FILE" | |
| - name: Check for changes | |
| id: git-check | |
| run: | | |
| # Show git status for debugging | |
| echo "Git status:" | |
| git status --porcelain | |
| # Check for changes in the specific directories we care about | |
| CHANGES=$(git status --porcelain src/packages/ docs/ 2>/dev/null || echo "") | |
| if [[ -z "$CHANGES" ]]; then | |
| echo "No changes detected in src/packages/ or docs/" | |
| echo "changes=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "Changes detected:" | |
| echo "$CHANGES" | |
| echo "changes=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Commit and push changes | |
| if: steps.git-check.outputs.changes == 'true' && steps.commit-message.outputs.should_commit == 'true' | |
| run: | | |
| SHOULD_COMMIT="${{ steps.commit-message.outputs.should_commit }}" | |
| TOTAL_UPDATED="${{ steps.commit-message.outputs.total_updated }}" | |
| echo "Should commit based on package updates: $SHOULD_COMMIT" | |
| echo "Total packages updated: $TOTAL_UPDATED" | |
| echo "Git changes detected: ${{ steps.git-check.outputs.changes }}" | |
| # Configure git for the action - use a PAT if available to trigger subsequent workflows | |
| if [ -n "${{ secrets.PAT_TOKEN }}" ]; then | |
| echo "Using PAT token to trigger subsequent workflows" | |
| git config --global user.name "chrisbbreuer" | |
| git config --global user.email "chrisbreuer93@gmail.com" | |
| git remote set-url origin https://x-access-token:${{ secrets.PAT_TOKEN }}@github.com/${{ github.repository }}.git | |
| else | |
| echo "Using default GitHub token (subsequent workflows may not trigger)" | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| fi | |
| # Add and commit changes | |
| git add src/packages/ docs/ | |
| git commit -m "${{ steps.commit-message.outputs.commit_message }}" -m "Automated update via GitHub Actions workflow" | |
| # Push changes | |
| git push origin main | |
| echo "Successfully committed and pushed changes" |