Improve performance of infer primary key #32303

Workflow file for this run

	# what?
	# Runs code quality checks, unit tests, integration tests and
	# verifies python build on all code commited to the repository. This workflow
	# should not require any secrets since it runs for PRs from forked repos. By
	# default, secrets are not passed to workflows running from a forked repos.

	# why?
	# Ensure code for dbt meets a certain quality standard.

	# when?
	# This will run for all PRs, when code is pushed to a release
	# branch, and when manually triggered.

	name: Tests and Code Checks

	on:
	push:
	branches:
	- "main"
	- "*.latest"
	- "releases/*"
	pull_request:
	workflow_dispatch:

	permissions: read-all

	# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
	concurrency:
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref \|\| github.sha }}
	cancel-in-progress: true

	defaults:
	run:
	shell: bash

	# top-level adjustments can be made here
	env:
	# number of parallel processes to spawn for python integration testing
	PYTHON_INTEGRATION_TEST_WORKERS: 5

	jobs:
	code-quality:
	name: code-quality

	runs-on: ubuntu-latest
	timeout-minutes: 10

	steps:
	- name: Check out the repository
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.8'

	- name: Install python dependencies
	run: \|
	python -m pip install --user --upgrade pip
	python -m pip --version
	make dev
	mypy --version
	dbt --version

	- name: Run pre-commit hooks
	run: pre-commit run --all-files --show-diff-on-failure

	unit:
	name: unit test / python ${{ matrix.python-version }}

	runs-on: ubuntu-latest
	timeout-minutes: 10

	strategy:
	fail-fast: false
	matrix:
	python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]

	env:
	TOXENV: "unit"

	steps:
	- name: Check out the repository
	uses: actions/checkout@v4

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install python dependencies
	run: \|
	python -m pip install --user --upgrade pip
	python -m pip --version
	python -m pip install tox
	tox --version

	- name: Run unit tests
	uses: nick-fields/retry@v3
	with:
	timeout_minutes: 10
	max_attempts: 3
	command: tox -e unit

	- name: Get current date
	if: always()
	id: date
	run: \|
	CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
	echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT

	- name: Upload Unit Test Coverage to Codecov
	if: ${{ matrix.python-version == '3.11' }}
	uses: codecov/codecov-action@v4
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	flags: unit

	integration-metadata:
	name: integration test metadata generation
	runs-on: ubuntu-latest
	outputs:
	split-groups: ${{ steps.generate-split-groups.outputs.split-groups }}
	include: ${{ steps.generate-include.outputs.include }}

	steps:
	- name: generate split-groups
	id: generate-split-groups
	run: \|
	MATRIX_JSON="["
	for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
	MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}")
	done
	MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}"
	MATRIX_JSON+="]"
	echo "split-groups=${MATRIX_JSON}"
	echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT

	- name: generate include
	id: generate-include
	run: \|
	INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' )
	INCLUDE_GROUPS="["
	for include in ${INCLUDE[@]}; do
	for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
	INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}")
	done
	done
	INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS \| sed 's/,*$//g')
	INCLUDE_GROUPS+="]"
	echo "include=${INCLUDE_GROUPS}"
	echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT

	integration:
	name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}

	runs-on: ${{ matrix.os }}
	timeout-minutes: 30
	needs:
	- integration-metadata
	strategy:
	fail-fast: false
	matrix:
	python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
	os: [ubuntu-20.04]
	split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
	include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
	env:
	TOXENV: integration
	DBT_INVOCATION_ENV: github-actions
	DBT_TEST_USER_1: dbt_test_user_1
	DBT_TEST_USER_2: dbt_test_user_2
	DBT_TEST_USER_3: dbt_test_user_3
	DD_CIVISIBILITY_AGENTLESS_ENABLED: true
	DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
	DD_SITE: datadoghq.com
	DD_ENV: ci
	DD_SERVICE: ${{ github.event.repository.name }}

	steps:
	- name: Check out the repository
	uses: actions/checkout@v4

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Set up postgres (linux)
	if: runner.os == 'Linux'
	uses: ./.github/actions/setup-postgres-linux

	- name: Set up postgres (macos)
	if: runner.os == 'macOS'
	uses: ./.github/actions/setup-postgres-macos

	- name: Set up postgres (windows)
	if: runner.os == 'Windows'
	uses: ./.github/actions/setup-postgres-windows

	- name: Install python tools
	run: \|
	python -m pip install --user --upgrade pip
	python -m pip --version
	python -m pip install tox
	tox --version

	- name: Run integration tests
	uses: nick-fields/retry@v3
	with:
	timeout_minutes: 30
	max_attempts: 3
	command: tox -- --ddtrace
	env:
	PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}

	- name: Get current date
	if: always()
	id: date
	run: \|
	CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
	echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT

	- uses: actions/upload-artifact@v4
	if: always()
	with:
	name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
	path: ./logs

	- name: Upload Integration Test Coverage to Codecov
	if: ${{ matrix.python-version == '3.11' }}
	uses: codecov/codecov-action@v4
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	flags: integration

	integration-report:
	if: ${{ always() }}
	name: Integration Test Suite
	runs-on: ubuntu-latest
	needs: integration
	steps:
	- name: "Integration Tests Failed"
	if: ${{ contains(needs.integration.result, 'failure') \|\| contains(needs.integration.result, 'cancelled') }}
	# when this is true the next step won't execute
	run: \|
	echo "::notice title='Integration test suite failed'"
	exit 1

	- name: "Integration Tests Passed"
	run: \|
	echo "::notice title='Integration test suite passed'"

	build:
	name: build packages

	runs-on: ubuntu-latest

	steps:
	- name: Check out the repository
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.8'

	- name: Install python dependencies
	run: \|
	python -m pip install --user --upgrade pip
	python -m pip install --upgrade setuptools wheel twine check-wheel-contents
	python -m pip --version

	- name: Build distributions
	run: ./scripts/build-dist.sh

	- name: Show distributions
	run: ls -lh dist/

	- name: Check distribution descriptions
	run: \|
	twine check dist/*

	- name: Check wheel contents
	run: \|
	check-wheel-contents dist/*.whl --ignore W007,W008

	- name: Install wheel distributions
	run: \|
	find ./dist/*.whl -maxdepth 1 -type f \| xargs python -m pip install --force-reinstall --find-links=dist/

	- name: Check wheel distributions
	run: \|
	dbt --version

	- name: Install source distributions
	# ignore dbt-1.0.0, which intentionally raises an error when installed from source
	run: \|
	find ./dist/*.gz -maxdepth 1 -type f \| xargs python -m pip install --force-reinstall --find-links=dist/

	- name: Check source distributions
	run: \|
	dbt --version

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Improve performance of infer primary key #32303

Workflow file

Improve performance of infer primary key #32303

Jobs

Run details

Workflow file for this run