From 6be778082b0b4601c32ae4b12c1350a88f01716a Mon Sep 17 00:00:00 2001 From: Francesco Leacche Date: Tue, 17 Jun 2025 17:44:12 +0100 Subject: [PATCH 1/3] generate config docs on ci --- .github/workflows/node-config-docsgen.yml | 196 ++++++++++++++++++++++ .github/workflows/stacks-core-tests.yml | 7 + 2 files changed, 203 insertions(+) create mode 100644 .github/workflows/node-config-docsgen.yml diff --git a/.github/workflows/node-config-docsgen.yml b/.github/workflows/node-config-docsgen.yml new file mode 100644 index 0000000000..de58f4fe0f --- /dev/null +++ b/.github/workflows/node-config-docsgen.yml @@ -0,0 +1,196 @@ +name: "Generate Node Configuration Documentation" + +on: + # Allow manual triggering + workflow_dispatch: + inputs: + output-dir: + description: "Output directory for generated documentation" + required: false + default: "./target/generated-docs" + type: string + min-doc-size: + description: "Minimum documentation size in bytes" + required: false + default: "50000" + type: string + rust-nightly-version: + description: "Specific nightly version to use (e.g., 'nightly-2025-06-17')" + required: false + default: "nightly-2025-06-17" + type: string + + # Allow being called by other workflows + workflow_call: + inputs: + output-dir: + description: "Output directory for generated documentation" + required: false + default: "./target/generated-docs" + type: string + min-doc-size: + description: "Minimum documentation size in bytes" + required: false + default: "50000" + type: string + rust-nightly-version: + description: "Specific nightly version to use (e.g., 'nightly-2025-06-17')" + required: false + default: "nightly-2025-06-17" + type: string + +jobs: + generate-config-docs: + name: Generate Configuration Documentation + runs-on: ubuntu-latest + + steps: + ## Checkout the code + - name: Checkout the latest code + id: git_checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + ## Install required system dependencies + - name: Install system dependencies + id: install_deps + shell: bash + run: | + echo "Installing system dependencies..." + sudo apt-get update -q + sudo apt-get install -y jq + + ## Install Rust nightly toolchain (required for rustdoc JSON output) + - name: Install Rust nightly toolchain + id: install_rust + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # stable + with: + toolchain: ${{ inputs.rust-nightly-version || 'nightly-2025-06-17' }} + components: rustdoc + + ## Cache Cargo dependencies + - name: Cache Cargo dependencies + id: cache_cargo + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./target/ + key: ${{ runner.os }}-cargo-nightly-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-nightly- + ${{ runner.os }}-cargo- + + ## Generate configuration documentation + - name: Generate Config Documentation + id: generate_docs + shell: bash + run: | + echo "Generating configuration documentation..." + + # Set environment variables for the script + export PROJECT_ROOT="$(pwd)" + export OUTPUT_DIR="$(pwd)/${{ inputs.output-dir || './target/generated-docs' }}" + + cd contrib/tools/config-docs-generator + chmod +x generate-config-docs.sh + ./generate-config-docs.sh + + # Verify output was generated + if [[ ! -f "$OUTPUT_DIR/node-parameters.md" ]]; then + echo "Error: Configuration documentation was not generated" + echo "Expected file: $OUTPUT_DIR/node-parameters.md" + exit 1 + fi + + echo "Documentation generated successfully at $OUTPUT_DIR/node-parameters.md" + + ## Validate generated documentation quality + - name: Validate Generated Documentation + id: validate_docs + shell: bash + run: | + echo "Validating generated documentation..." + + MARKDOWN_FILE="${{ inputs.output-dir || './target/generated-docs' }}/node-parameters.md" + + # Check if file exists and has content + if [[ ! -f "$MARKDOWN_FILE" ]]; then + echo "Error: Markdown file not found at $MARKDOWN_FILE" + exit 1 + fi + + # Check file size (should be substantial) + FILE_SIZE=$(wc -c < "$MARKDOWN_FILE") + MIN_SIZE=${{ inputs.min-doc-size || '50000' }} + + if [[ $FILE_SIZE -lt $MIN_SIZE ]]; then + echo "Error: Generated documentation is too small ($FILE_SIZE bytes, minimum $MIN_SIZE)" + echo "This likely indicates a generation failure" + exit 1 + fi + + # Check word count and basic structure + WORD_COUNT=$(wc -w < "$MARKDOWN_FILE") + LINE_COUNT=$(wc -l < "$MARKDOWN_FILE") + + echo "Documentation validation results:" + echo " - File size: $FILE_SIZE bytes" + echo " - Word count: $WORD_COUNT words" + echo " - Line count: $LINE_COUNT lines" + + # Basic content validation + if ! grep -q "# Configuration Reference" "$MARKDOWN_FILE"; then + echo "Warning: Documentation may be malformed - missing main title" + fi + + if [[ $WORD_COUNT -lt 100 ]]; then + echo "Warning: Documentation seems very short ($WORD_COUNT words)" + fi + + echo "Documentation validation completed successfully ✅" + + ## Upload documentation as artifact + - name: Upload Documentation Artifact + id: upload_artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: "node-parameters.md" + path: | + ${{ inputs.output-dir || './target/generated-docs' }}/* + retention-days: 30 + if-no-files-found: error + + ## Generate job summary + - name: Generate Job Summary + id: summary + shell: bash + run: | + echo "## Configuration Documentation Generated ✅" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The Stacks Core configuration documentation has been successfully generated and uploaded." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Add file statistics if available + MARKDOWN_FILE="${{ inputs.output-dir || './target/generated-docs' }}/node-parameters.md" + if [[ -f "$MARKDOWN_FILE" ]]; then + FILE_SIZE=$(wc -c < "$MARKDOWN_FILE") + WORD_COUNT=$(wc -w < "$MARKDOWN_FILE") + + echo "### 📊 Generation Statistics:" >> $GITHUB_STEP_SUMMARY + echo "- **File Size**: $(numfmt --to=iec-i --suffix=B $FILE_SIZE)" >> $GITHUB_STEP_SUMMARY + echo "- **Word Count**: $WORD_COUNT words" >> $GITHUB_STEP_SUMMARY + echo "- **Rust Toolchain**: ${{ inputs.rust-nightly-version || 'nightly-2025-06-17' }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + echo "### 📄 Generated Files:" >> $GITHUB_STEP_SUMMARY + echo "- \`node-parameters.md\` - Complete configuration documentation" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### 📦 Artifact Details:" >> $GITHUB_STEP_SUMMARY + echo "- **Name**: \`node-parameters.md\`" >> $GITHUB_STEP_SUMMARY + echo "- **Retention**: 30 days" >> $GITHUB_STEP_SUMMARY + echo "- **Location**: Available in workflow run artifacts section" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "> 💡 **Tip**: Download the artifact to access the generated documentation files" diff --git a/.github/workflows/stacks-core-tests.yml b/.github/workflows/stacks-core-tests.yml index 05b9f09f62..7144db92ec 100644 --- a/.github/workflows/stacks-core-tests.yml +++ b/.github/workflows/stacks-core-tests.yml @@ -68,6 +68,12 @@ jobs: input: ./docs/rpc/openapi.yaml output: ./open-api-docs.html + ## Generate and upload node configuration documentation + node-config-docsgen: + name: Node Configuration Documentation + needs: open-api-validation + uses: ./.github/workflows/node-config-docsgen.yml + ## Disabled ## - this test can take several hours to run nettest: @@ -138,6 +144,7 @@ jobs: if: always() needs: - open-api-validation + - node-config-docsgen - core-contracts-clarinet-test steps: - name: Check Tests Status From 110753b8bf440bd517a18f348e9aeff1f2eb4972 Mon Sep 17 00:00:00 2001 From: Francesco Leacche Date: Wed, 25 Jun 2025 16:49:39 +0100 Subject: [PATCH 2/3] add node-config-docsgen action --- .github/workflows/node-config-docsgen.yml | 196 ---------------------- .github/workflows/stacks-core-tests.yml | 3 +- 2 files changed, 1 insertion(+), 198 deletions(-) delete mode 100644 .github/workflows/node-config-docsgen.yml diff --git a/.github/workflows/node-config-docsgen.yml b/.github/workflows/node-config-docsgen.yml deleted file mode 100644 index de58f4fe0f..0000000000 --- a/.github/workflows/node-config-docsgen.yml +++ /dev/null @@ -1,196 +0,0 @@ -name: "Generate Node Configuration Documentation" - -on: - # Allow manual triggering - workflow_dispatch: - inputs: - output-dir: - description: "Output directory for generated documentation" - required: false - default: "./target/generated-docs" - type: string - min-doc-size: - description: "Minimum documentation size in bytes" - required: false - default: "50000" - type: string - rust-nightly-version: - description: "Specific nightly version to use (e.g., 'nightly-2025-06-17')" - required: false - default: "nightly-2025-06-17" - type: string - - # Allow being called by other workflows - workflow_call: - inputs: - output-dir: - description: "Output directory for generated documentation" - required: false - default: "./target/generated-docs" - type: string - min-doc-size: - description: "Minimum documentation size in bytes" - required: false - default: "50000" - type: string - rust-nightly-version: - description: "Specific nightly version to use (e.g., 'nightly-2025-06-17')" - required: false - default: "nightly-2025-06-17" - type: string - -jobs: - generate-config-docs: - name: Generate Configuration Documentation - runs-on: ubuntu-latest - - steps: - ## Checkout the code - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - ## Install required system dependencies - - name: Install system dependencies - id: install_deps - shell: bash - run: | - echo "Installing system dependencies..." - sudo apt-get update -q - sudo apt-get install -y jq - - ## Install Rust nightly toolchain (required for rustdoc JSON output) - - name: Install Rust nightly toolchain - id: install_rust - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # stable - with: - toolchain: ${{ inputs.rust-nightly-version || 'nightly-2025-06-17' }} - components: rustdoc - - ## Cache Cargo dependencies - - name: Cache Cargo dependencies - id: cache_cargo - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ./target/ - key: ${{ runner.os }}-cargo-nightly-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-nightly- - ${{ runner.os }}-cargo- - - ## Generate configuration documentation - - name: Generate Config Documentation - id: generate_docs - shell: bash - run: | - echo "Generating configuration documentation..." - - # Set environment variables for the script - export PROJECT_ROOT="$(pwd)" - export OUTPUT_DIR="$(pwd)/${{ inputs.output-dir || './target/generated-docs' }}" - - cd contrib/tools/config-docs-generator - chmod +x generate-config-docs.sh - ./generate-config-docs.sh - - # Verify output was generated - if [[ ! -f "$OUTPUT_DIR/node-parameters.md" ]]; then - echo "Error: Configuration documentation was not generated" - echo "Expected file: $OUTPUT_DIR/node-parameters.md" - exit 1 - fi - - echo "Documentation generated successfully at $OUTPUT_DIR/node-parameters.md" - - ## Validate generated documentation quality - - name: Validate Generated Documentation - id: validate_docs - shell: bash - run: | - echo "Validating generated documentation..." - - MARKDOWN_FILE="${{ inputs.output-dir || './target/generated-docs' }}/node-parameters.md" - - # Check if file exists and has content - if [[ ! -f "$MARKDOWN_FILE" ]]; then - echo "Error: Markdown file not found at $MARKDOWN_FILE" - exit 1 - fi - - # Check file size (should be substantial) - FILE_SIZE=$(wc -c < "$MARKDOWN_FILE") - MIN_SIZE=${{ inputs.min-doc-size || '50000' }} - - if [[ $FILE_SIZE -lt $MIN_SIZE ]]; then - echo "Error: Generated documentation is too small ($FILE_SIZE bytes, minimum $MIN_SIZE)" - echo "This likely indicates a generation failure" - exit 1 - fi - - # Check word count and basic structure - WORD_COUNT=$(wc -w < "$MARKDOWN_FILE") - LINE_COUNT=$(wc -l < "$MARKDOWN_FILE") - - echo "Documentation validation results:" - echo " - File size: $FILE_SIZE bytes" - echo " - Word count: $WORD_COUNT words" - echo " - Line count: $LINE_COUNT lines" - - # Basic content validation - if ! grep -q "# Configuration Reference" "$MARKDOWN_FILE"; then - echo "Warning: Documentation may be malformed - missing main title" - fi - - if [[ $WORD_COUNT -lt 100 ]]; then - echo "Warning: Documentation seems very short ($WORD_COUNT words)" - fi - - echo "Documentation validation completed successfully ✅" - - ## Upload documentation as artifact - - name: Upload Documentation Artifact - id: upload_artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: "node-parameters.md" - path: | - ${{ inputs.output-dir || './target/generated-docs' }}/* - retention-days: 30 - if-no-files-found: error - - ## Generate job summary - - name: Generate Job Summary - id: summary - shell: bash - run: | - echo "## Configuration Documentation Generated ✅" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The Stacks Core configuration documentation has been successfully generated and uploaded." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Add file statistics if available - MARKDOWN_FILE="${{ inputs.output-dir || './target/generated-docs' }}/node-parameters.md" - if [[ -f "$MARKDOWN_FILE" ]]; then - FILE_SIZE=$(wc -c < "$MARKDOWN_FILE") - WORD_COUNT=$(wc -w < "$MARKDOWN_FILE") - - echo "### 📊 Generation Statistics:" >> $GITHUB_STEP_SUMMARY - echo "- **File Size**: $(numfmt --to=iec-i --suffix=B $FILE_SIZE)" >> $GITHUB_STEP_SUMMARY - echo "- **Word Count**: $WORD_COUNT words" >> $GITHUB_STEP_SUMMARY - echo "- **Rust Toolchain**: ${{ inputs.rust-nightly-version || 'nightly-2025-06-17' }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - - echo "### 📄 Generated Files:" >> $GITHUB_STEP_SUMMARY - echo "- \`node-parameters.md\` - Complete configuration documentation" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### 📦 Artifact Details:" >> $GITHUB_STEP_SUMMARY - echo "- **Name**: \`node-parameters.md\`" >> $GITHUB_STEP_SUMMARY - echo "- **Retention**: 30 days" >> $GITHUB_STEP_SUMMARY - echo "- **Location**: Available in workflow run artifacts section" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "> 💡 **Tip**: Download the artifact to access the generated documentation files" diff --git a/.github/workflows/stacks-core-tests.yml b/.github/workflows/stacks-core-tests.yml index 7144db92ec..ececd229d7 100644 --- a/.github/workflows/stacks-core-tests.yml +++ b/.github/workflows/stacks-core-tests.yml @@ -71,8 +71,7 @@ jobs: ## Generate and upload node configuration documentation node-config-docsgen: name: Node Configuration Documentation - needs: open-api-validation - uses: ./.github/workflows/node-config-docsgen.yml + uses: stacks-network/actions/node-config-docsgen@main ## Disabled ## - this test can take several hours to run From 3ba45fd54ef98c1bf353ad052449a96e09ff787f Mon Sep 17 00:00:00 2001 From: Francesco Leacche Date: Wed, 25 Jun 2025 16:50:18 +0100 Subject: [PATCH 3/3] add to work with action --- .../generate-config-docs.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/contrib/tools/config-docs-generator/generate-config-docs.sh b/contrib/tools/config-docs-generator/generate-config-docs.sh index b9c0924512..dea9459ac1 100755 --- a/contrib/tools/config-docs-generator/generate-config-docs.sh +++ b/contrib/tools/config-docs-generator/generate-config-docs.sh @@ -12,7 +12,7 @@ NC='\033[0m' # No Color SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="${PROJECT_ROOT:-$(cd "$SCRIPT_DIR/../../../" && pwd)}" CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PROJECT_ROOT/target}" -OUTPUT_DIR="${OUTPUT_DIR:-$CARGO_TARGET_DIR/generated-docs}" +OUTPUT_FILE="${OUTPUT_FILE:-$CARGO_TARGET_DIR/generated-docs/node-parameters.md}" TEMP_DIR="${TEMP_DIR:-$CARGO_TARGET_DIR/doc-generation}" # Binary paths - allow override via environment @@ -52,7 +52,7 @@ main() { log_info "Starting config documentation generation..." # Create necessary directories - mkdir -p "$OUTPUT_DIR" + mkdir -p "$(dirname "$OUTPUT_FILE")" mkdir -p "$TEMP_DIR" cd "$PROJECT_ROOT" @@ -81,22 +81,21 @@ main() { # Step 3: Generate Markdown log_info "Generating Markdown documentation..." - MARKDOWN_OUTPUT="$OUTPUT_DIR/configuration-reference.md" # Call the command - "$GENERATE_MARKDOWN_BIN" --input "$EXTRACTED_JSON" --output "$MARKDOWN_OUTPUT" --template "$TEMPLATE_PATH" --section-name-mappings "$SECTION_MAPPINGS_PATH" + "$GENERATE_MARKDOWN_BIN" --input "$EXTRACTED_JSON" --output "$OUTPUT_FILE" --template "$TEMPLATE_PATH" --section-name-mappings "$SECTION_MAPPINGS_PATH" log_info "Documentation generation complete!" log_info "Generated files:" - log_info " - Configuration reference: $MARKDOWN_OUTPUT" + log_info " - Configuration reference: $OUTPUT_FILE" log_info " - Intermediate JSON: $EXTRACTED_JSON" # Verify output - if [[ -f "$MARKDOWN_OUTPUT" ]]; then - WORD_COUNT=$(wc -w < "$MARKDOWN_OUTPUT") + if [[ -f "$OUTPUT_FILE" ]]; then + WORD_COUNT=$(wc -w < "$OUTPUT_FILE") log_info "Generated Markdown contains $WORD_COUNT words" else - log_error "Expected output file not found: $MARKDOWN_OUTPUT" + log_error "Expected output file not found: $OUTPUT_FILE" exit 1 fi } @@ -125,7 +124,7 @@ DESCRIPTION: Source file: stackslib/src/config/mod.rs OUTPUT: - docs/generated/configuration-reference.md + docs/generated/node-parameters.md EOF }