Sample Data: document, update scripts to validate, update tests #59
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate-Samples | |
| on: | |
| push: | |
| paths: | |
| - 'samples/*/TIDES/*' | |
| - 'spec/*' | |
| pull_request: | |
| paths: | |
| - 'samples/*/TIDES/*' | |
| - 'spec/*' | |
| workflow_dispatch: | |
| create: | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.x | |
| - name: Install Frictionless | |
| run: pip install frictionless[excel,json]>=5.0.0 | |
| - name: Find Sample Folders | |
| id: sample_folders | |
| run: | | |
| pwd | |
| samples=$(find ./samples -type d -name 'TIDES') | |
| echo $samples | |
| echo "samples=${samples}" >> $GITHUB_OUTPUT | |
| - name: Validate Sample Data | |
| id: validation | |
| run: | | |
| samples=(${{ steps.sample_folders.outputs.samples }}) | |
| results="" | |
| for sample in "${samples[@]}"; do | |
| result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) | |
| results+="$result\n\n" | |
| done | |
| echo "RESULTS<<EOF" >> $GITHUB_ENV | |
| echo ${results} >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "$GITHUB_ENV" | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v4 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| console.log("Begin sample construction") | |
| const samples = '${{ env.samples }}'.split('\n'); | |
| const results = '${{ env.results }}'.split('\n\n'); | |
| let comment = `**Data Validation Report**\n\n`; | |
| comment += '| Sample | Status |\n'; | |
| comment += '| ------ | ------ |\n'; | |
| console.log("Constructed header") | |
| for (let i = 0; i < samples.length; i++) { | |
| console.log(samples[i]) | |
| console.log(results[i]) | |
| const sample = samples[i]; | |
| const result = JSON.parse(results[i]); | |
| let status = ''; | |
| if (result.valid) { | |
| status = ':heavy_check_mark:'; | |
| } else if (result.stats.errorCount > 0) { | |
| status = ':x:'; | |
| } else { | |
| status = ':warning:'; | |
| } | |
| comment += `| ${sample} | ${status} |\n`; | |
| } | |
| github.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); |