[CI] Split pytest into groups (#5943) #1791
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: On push | |
| on: | |
| push: | |
| branches: | |
| - py3 | |
| concurrency: deploy-${{ github.ref }} | |
| jobs: | |
| py3-type: | |
| name: "[py3] Type Check" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -r src/requirements.txt | |
| - name: Run Type Check | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/typecheck_py3.sh | |
| py3-lint: | |
| name: "[py3] Format & Lint" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -r src/requirements.txt | |
| - name: Run Format & Lint | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/lint_py3.sh | |
| py3-test: | |
| name: "[py3] Test" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| group: [1, 2, 3] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -r src/requirements.txt | |
| - name: Run Tests | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/test_py3.sh --cov --splits 3 --group ${{ matrix.group }} | |
| - name: Save coverage artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage${{ matrix.group }} | |
| path: .coverage | |
| py3-coverage: | |
| name: "[py3] Coverage" | |
| runs-on: ubuntu-latest | |
| needs: [py3-test] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Combine coverage | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| coverage combine coverage*/.coverage* | |
| coverage xml | |
| - name: Upload coverage to Codecov | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| uses: codecov/codecov-action@v4.4.0 | |
| node-lint: | |
| name: "[node] Format & Lint" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Node | |
| uses: actions/setup-node@v4.0.2 | |
| with: | |
| node-version: "18" | |
| cache: "npm" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: npm ci | |
| - name: Run Format & Lint | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/lint_node.sh | |
| node-test: | |
| name: "[node] Test" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Node | |
| uses: actions/setup-node@v4.0.2 | |
| with: | |
| node-version: "18" | |
| cache: "npm" | |
| - name: Install Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: npm ci | |
| - name: Run Tests | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/test_node.sh | |
| - name: Upload coverage to Codecov | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| uses: codecov/codecov-action@v4.4.0 | |
| with: | |
| directory: ./.node_coverage/ | |
| bash-lint: | |
| name: "[bash] Lint" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install Shellcheck | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install shellcheck-py | |
| - name: Install shfmt | |
| run: | | |
| mkdir -p $HOME/.local/bin | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| wget "https://github.com/mvdan/sh/releases/download/v3.2.2/shfmt_v3.2.2_linux_amd64" -O $HOME/.local/bin/shfmt | |
| chmod +x $HOME/.local/bin/shfmt | |
| - name: Run Lint | |
| run: ./ops/lint_bash.sh | |
| build-web: | |
| name: Build Web | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up Node | |
| uses: actions/setup-node@v4.0.2 | |
| with: | |
| node-version: "18" | |
| cache: "npm" | |
| - name: Install Node Dependencies | |
| run: npm ci | |
| - name: Run Build | |
| run: ./ops/build/run_buildweb.sh --env | |
| env: | |
| GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| FIREBASE_API_KEY: ${{ secrets.FIREBASE_API_KEY }} | |
| FIREBASE_APP_ID: ${{ secrets.FIREBASE_APP_ID }} | |
| FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.FIREBASE_MESSAGING_SENDER_ID }} | |
| - name: Archive production artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: web-build | |
| path: src/build | |
| ops-test: | |
| name: Ops Fullstack Test | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Python Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install requests | |
| - name: Set up Node | |
| uses: actions/setup-node@v4.0.2 | |
| with: | |
| node-version: "18" | |
| cache: "npm" | |
| - name: Install Node Dependencies | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: npm ci | |
| - name: Install Vagrant | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install vagrant | |
| - name: Create keys file | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: cp ./src/backend/web/static/javascript/tba_js/tba_keys_template.js ./src/backend/web/static/javascript/tba_js/tba_keys.js | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: network=host | |
| - name: Build Docker Image | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| uses: docker/build-push-action@v5.1.0 | |
| with: | |
| builder: ${{ steps.buildx.outputs.name }} | |
| context: ./ops/dev/docker | |
| push: true | |
| tags: localhost:5000/tba-py3-dev:latest | |
| cache-from: | | |
| type=gha | |
| ghcr.io/the-blue-alliance/the-blue-alliance/tba-py3-dev:latest | |
| cache-to: type=gha,mode=max | |
| - name: Start Container | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: vagrant up | |
| - name: Test Vagrant Startup | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/test_vagrant_startup.py | |
| - name: Test Ops Fullstack | |
| if: contains(github.event.commits[0].message, '[clowntown]') == false | |
| run: ./ops/test_ops.sh | |
| deploy-queues: | |
| name: Deploy Queues | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: gcloud app deploy src/queue.yaml | |
| deploy-index: | |
| name: Deploy Index | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: gcloud app deploy src/index.yaml | |
| deploy-default: | |
| name: Deploy Default Service | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_module.sh src/default.yaml | |
| deploy-web: | |
| name: Deploy Web Service | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| py3-type, | |
| py3-lint, | |
| py3-test, | |
| node-lint, | |
| node-test, | |
| build-web, | |
| ops-test, | |
| deploy-queues, | |
| deploy-index, | |
| ] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Download production artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: web-build | |
| path: src/build | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_module.sh src/web.yaml | |
| deploy-api: | |
| name: Deploy API Service | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test, ops-test, deploy-queues, deploy-index] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_module.sh src/api.yaml | |
| deploy-tasks-io: | |
| name: Deploy Tasks IO Service | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test, ops-test, deploy-queues, deploy-index] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_module.sh src/tasks_io.yaml | |
| deploy-tasks-cpu: | |
| name: Deploy Tasks CPU Service | |
| runs-on: ubuntu-latest | |
| needs: [py3-type, py3-lint, py3-test, ops-test, deploy-queues, deploy-index] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_module.sh src/tasks_cpu.yaml | |
| deploy-cron: | |
| name: Deploy Cron Jobs | |
| runs-on: ubuntu-latest | |
| needs: [deploy-tasks-io, deploy-tasks-cpu] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: gcloud app deploy src/cron.yaml | |
| deploy-dispatch: | |
| name: Deploy Dispatch | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| deploy-queues, | |
| deploy-index, | |
| deploy-web, | |
| deploy-api, | |
| deploy-tasks-io, | |
| deploy-tasks-cpu, | |
| ] | |
| if: contains(github.event.commits[0].message, '[nodeploy]') == false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCLOUD_AUTH }} | |
| - name: Setup Google Cloud Platform | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| version: "latest" | |
| project_id: ${{ secrets.GCLOUD_PROJECT_ID }} | |
| - name: Deploy | |
| run: ./ops/deploy/deploy_dispatch.sh |