Scrape and Format Data #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape and Format Data | |
| on: | |
| schedule: | |
| - cron: "0 23 * * *" # Daily at 11 PM UTC (adjust as needed) | |
| workflow_dispatch: | |
| inputs: | |
| use-scrape-cache: | |
| type: boolean | |
| description: Use OpenStates scraper cache | |
| default: false | |
| force-update: | |
| type: boolean | |
| description: "Force push even if upstream changed (use with caution)" | |
| default: false | |
| env: | |
| STATE_CODE: ks # ⚠️ UPDATE THIS: Change to your state code (e.g., wy, usa, il, tx) | |
| jobs: | |
| scrape: | |
| name: "🕷️ Scrape Data" | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| outputs: | |
| scrape-artifact-name: ${{ steps.scrape.outputs.scrape-artifact-name }} | |
| scrape-tarball-path: ${{ steps.scrape.outputs.scrape-tarball-path }} | |
| manifest-path: ${{ steps.scrape.outputs.manifest-path }} | |
| steps: | |
| - name: Checkout state repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Run scraper action | |
| id: scrape | |
| uses: windy-civi/toolkit/actions/scrape@main | |
| with: | |
| state: ${{ env.STATE_CODE }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| use-scrape-cache: ${{ inputs.use-scrape-cache }} | |
| format: | |
| name: "📝 Format Data" | |
| runs-on: ubuntu-latest | |
| needs: scrape | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout state repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download scrape artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.scrape.outputs.scrape-artifact-name }} | |
| - name: Run formatter action | |
| uses: windy-civi/toolkit/actions/format@main | |
| with: | |
| state: ${{ env.STATE_CODE }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| scrape-artifact-name: ${{ needs.scrape.outputs.scrape-artifact-name }} | |
| force-update: ${{ inputs.force-update }} |