From e744a66ed1ea76e75f46c2ef12f9f0f9ed33f9ac Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 27 Nov 2024 12:47:29 -0700 Subject: [PATCH 1/7] Add dbt version requirement --- dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_project.yml b/dbt_project.yml index a2c40f4..dd0a8cd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,7 @@ name: 'the_tuva_project_demo' version: '0.1.8' config-version: 2 +require-dbt-version: ">=1.8.6" profile: default From 60630d5d411ed288fe903a95f0503cedd7bb29a0 Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 27 Nov 2024 12:53:48 -0700 Subject: [PATCH 2/7] Remove old github workflows --- ...dbt_v1.4.6_bigquery_build_full_refresh.yml | 36 ---------------- ...dbt_v1.4.6_redshift_build_full_refresh.yml | 38 ----------------- ...bt_v1.4.6_snowflake_build_full_refresh.yml | 41 ------------------- 3 files changed, 115 deletions(-) delete mode 100644 .github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml delete mode 100644 .github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml delete mode 100644 .github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml diff --git a/.github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml deleted file mode 100644 index 5979664..0000000 --- a/.github/workflows/dbt_v1.4.6_bigquery_build_full_refresh.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: dbt_v1.4.6_bigquery_build_full_refresh - -on: - workflow_dispatch: - - -env: - DBT_BIGQUERY_TOKEN: ${{ secrets.DBT_BIGQUERY_CI_TOKEN }} - DBT_BIGQUERY_CI_PROJECT: ${{ secrets.DBT_BIGQUERY_CI_PROJECT }} - -jobs: - action: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: dbt-deps - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/bigquery" - - - name: dbt-debug - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/bigquery" - - - name: dbt-build - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery --vars {"tuva_schema_prefix":"demo"}' - - - name: Get the result - if: ${{ always() }} - run: echo "${{ steps.dbt-build.outputs.result }}" - shell: bash diff --git a/.github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml deleted file mode 100644 index 5e568eb..0000000 --- a/.github/workflows/dbt_v1.4.6_redshift_build_full_refresh.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: dbt_v1.4.6_redshift_build_full_refresh - -on: - workflow_dispatch: - - -env: - DBT_REDSHIFT_HOST: ${{ secrets.DBT_REDSHIFT_HOST }} - DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }} - DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }} - DBT_REDSHIFT_CI_DATABASE: ${{ secrets.DBT_REDSHIFT_CI_DATABASE }} - -jobs: - action: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: dbt-deps - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/redshift" - - - name: dbt-debug - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/redshift" - - - name: dbt-build - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --vars {"tuva_schema_prefix":"demo"}' - - - name: Get the result - if: ${{ always() }} - run: echo "${{ steps.dbt-build.outputs.result }}" - shell: bash diff --git a/.github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml b/.github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml deleted file mode 100644 index 46d80e8..0000000 --- a/.github/workflows/dbt_v1.4.6_snowflake_build_full_refresh.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: dbt_v1.4.6_snowflake_build_full_refresh - -on: - workflow_dispatch: - - -env: - DBT_SNOWFLAKE_DEV_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_DEV_ACCOUNT }} - DBT_SNOWFLAKE_DEV_CI_DATABASE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_DATABASE }} - DBT_SNOWFLAKE_DEV_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_PASSWORD }} - DBT_SNOWFLAKE_DEV_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_ROLE }} - DBT_SNOWFLAKE_DEV_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_SCHEMA }} - DBT_SNOWFLAKE_DEV_CI_USER: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_USER }} - DBT_SNOWFLAKE_DEV_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_WAREHOUSE }} - -jobs: - action: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: dbt-deps - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_dev --target ci" - - - name: dbt-debug - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_dev --target ci" - - - name: dbt-build - uses: mwhitaker/dbt-action@v1.4.6 - with: - dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_dev --target ci --vars {"tuva_schema_prefix":"demo"}' - - - name: Get the result - if: ${{ always() }} - run: echo "${{ steps.dbt-build.outputs.result }}" - shell: bash From 69c085c60241e697517e5829d45e8d3846b0408d Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 27 Nov 2024 12:54:03 -0700 Subject: [PATCH 3/7] Update dbt version in github workflows --- ...esh.yml => dbt_v1.8.6_bigquery_build_full_refresh.yml} | 8 ++++---- ...esh.yml => dbt_v1.8.6_redshift_build_full_refresh.yml} | 8 ++++---- ...sh.yml => dbt_v1.8.6_snowflake_build_full_refresh.yml} | 8 ++++---- ...eekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml} | 8 ++++---- ...ekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml} | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) rename .github/workflows/{dbt_v1.6.3_bigquery_build_full_refresh.yml => dbt_v1.8.6_bigquery_build_full_refresh.yml} (83%) rename .github/workflows/{dbt_v1.6.3_redshift_build_full_refresh.yml => dbt_v1.8.6_redshift_build_full_refresh.yml} (85%) rename .github/workflows/{dbt_v1.6.3_snowflake_build_full_refresh.yml => dbt_v1.8.6_snowflake_build_full_refresh.yml} (87%) rename .github/workflows/{weekly_dbt_v1.6.3_snowflake_dev_build_full_refresh.yml => weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml} (87%) rename .github/workflows/{weekly_dbt_v1.6.3_snowflake_prod_build_full_refresh.yml => weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml} (87%) diff --git a/.github/workflows/dbt_v1.6.3_bigquery_build_full_refresh.yml b/.github/workflows/dbt_v1.8.6_bigquery_build_full_refresh.yml similarity index 83% rename from .github/workflows/dbt_v1.6.3_bigquery_build_full_refresh.yml rename to .github/workflows/dbt_v1.8.6_bigquery_build_full_refresh.yml index 580e684..4d341a3 100644 --- a/.github/workflows/dbt_v1.6.3_bigquery_build_full_refresh.yml +++ b/.github/workflows/dbt_v1.8.6_bigquery_build_full_refresh.yml @@ -1,4 +1,4 @@ -name: dbt_v1.6.3_bigquery_build_full_refresh +name: dbt_v1.8.6_bigquery_build_full_refresh on: workflow_dispatch: @@ -18,17 +18,17 @@ jobs: - uses: actions/checkout@v2 - name: dbt-deps - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/bigquery --target dev" - name: dbt-debug - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/bigquery --target dev" - name: dbt-build - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/bigquery --target dev' diff --git a/.github/workflows/dbt_v1.6.3_redshift_build_full_refresh.yml b/.github/workflows/dbt_v1.8.6_redshift_build_full_refresh.yml similarity index 85% rename from .github/workflows/dbt_v1.6.3_redshift_build_full_refresh.yml rename to .github/workflows/dbt_v1.8.6_redshift_build_full_refresh.yml index afe2b35..cf88bd2 100644 --- a/.github/workflows/dbt_v1.6.3_redshift_build_full_refresh.yml +++ b/.github/workflows/dbt_v1.8.6_redshift_build_full_refresh.yml @@ -1,4 +1,4 @@ -name: dbt_v1.6.3_redshift_build_full_refresh +name: dbt_v1.8.6_redshift_build_full_refresh on: workflow_dispatch: @@ -20,17 +20,17 @@ jobs: - uses: actions/checkout@v2 - name: dbt-deps - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/redshift --target dev" - name: dbt-debug - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/redshift --target dev" - name: dbt-build - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/redshift --target dev' diff --git a/.github/workflows/dbt_v1.6.3_snowflake_build_full_refresh.yml b/.github/workflows/dbt_v1.8.6_snowflake_build_full_refresh.yml similarity index 87% rename from .github/workflows/dbt_v1.6.3_snowflake_build_full_refresh.yml rename to .github/workflows/dbt_v1.8.6_snowflake_build_full_refresh.yml index fa9ba4a..cd370f2 100644 --- a/.github/workflows/dbt_v1.6.3_snowflake_build_full_refresh.yml +++ b/.github/workflows/dbt_v1.8.6_snowflake_build_full_refresh.yml @@ -1,4 +1,4 @@ -name: dbt_v1.6.3_snowflake_build_full_refresh +name: dbt_v1.8.6_snowflake_build_full_refresh on: workflow_dispatch: @@ -23,17 +23,17 @@ jobs: - uses: actions/checkout@v2 - name: dbt-deps - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_dev" - name: dbt-debug - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_dev" - name: dbt-build - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: 'dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_dev' diff --git a/.github/workflows/weekly_dbt_v1.6.3_snowflake_dev_build_full_refresh.yml b/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml similarity index 87% rename from .github/workflows/weekly_dbt_v1.6.3_snowflake_dev_build_full_refresh.yml rename to .github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml index d28ee1d..be40118 100644 --- a/.github/workflows/weekly_dbt_v1.6.3_snowflake_dev_build_full_refresh.yml +++ b/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml @@ -1,4 +1,4 @@ -name: weekly_dbt_v1.6.3_snowflake_dev_build_full_refresh +name: weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh on: workflow_dispatch: @@ -23,17 +23,17 @@ jobs: - uses: actions/checkout@v2 - name: dbt-deps - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" - name: dbt-debug - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" - name: dbt-build - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" diff --git a/.github/workflows/weekly_dbt_v1.6.3_snowflake_prod_build_full_refresh.yml b/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml similarity index 87% rename from .github/workflows/weekly_dbt_v1.6.3_snowflake_prod_build_full_refresh.yml rename to .github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml index b037631..21f632d 100644 --- a/.github/workflows/weekly_dbt_v1.6.3_snowflake_prod_build_full_refresh.yml +++ b/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml @@ -1,4 +1,4 @@ -name: weekly_dbt_v1.6.3_snowflake_prod_build_full_refresh +name: weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh on: workflow_dispatch: @@ -23,17 +23,17 @@ jobs: - uses: actions/checkout@v2 - name: dbt-deps - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_prod" - name: dbt-debug - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_prod" - name: dbt-build - uses: mwhitaker/dbt-action@v1.6.3 + uses: mwhitaker/dbt-action@v1.8.6 with: dbt_command: "dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_prod" From daf0179844c3774c09d5ab6ab189944a9579a49b Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 18 Dec 2024 09:45:41 -0700 Subject: [PATCH 4/7] Add fabric profile --- integration_tests/profiles/fabric/profiles.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 integration_tests/profiles/fabric/profiles.yml diff --git a/integration_tests/profiles/fabric/profiles.yml b/integration_tests/profiles/fabric/profiles.yml new file mode 100644 index 0000000..6c7a73c --- /dev/null +++ b/integration_tests/profiles/fabric/profiles.yml @@ -0,0 +1,14 @@ +default: + target: dev + outputs: + dev: + type: fabric + host: "{{ env_var('DBT_FABRIC_CI_HOST') }}" + database: "{{ env_var('DBT_FABRIC_CI_DATABASE') }}" + schema: "{{ env_var('DBT_FABRIC_CI_SCHEMA') }}" + authentication: ServicePrincipal + driver: ODBC Driver 18 for SQL Server + client_id: "{{ env_var('DBT_FABRIC_SERVICE_PRINCIPAL_ID') }}" + client_secret: "{{ env_var('DBT_FABRIC_SERVICE_PRINCIPAL_SECRET') }}" + tenant_id: "{{ env_var('DBT_FABRIC_TENANT_ID') }}" + threads: 8 From e4430d5d60c6e4215bb29e3985edc044e10e1a9a Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 18 Dec 2024 09:46:09 -0700 Subject: [PATCH 5/7] Update threads in profiles --- integration_tests/profiles/redshift/profiles.yml | 4 ++-- integration_tests/profiles/snowflake_dev/profiles.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/profiles/redshift/profiles.yml b/integration_tests/profiles/redshift/profiles.yml index 29040b5..4ccdc9a 100644 --- a/integration_tests/profiles/redshift/profiles.yml +++ b/integration_tests/profiles/redshift/profiles.yml @@ -9,7 +9,7 @@ default: port: 5439 ra3_node: true schema: default - threads: 2 + threads: 4 type: redshift user: "{{ env_var('DBT_REDSHIFT_CI_USER') }}" ci: @@ -21,7 +21,7 @@ default: port: 5439 ra3_node: true schema: default - threads: 2 + threads: 4 type: redshift user: "{{ env_var('DBT_REDSHIFT_CI_USER') }}" target: dev diff --git a/integration_tests/profiles/snowflake_dev/profiles.yml b/integration_tests/profiles/snowflake_dev/profiles.yml index 2d1e7f6..1a693e3 100644 --- a/integration_tests/profiles/snowflake_dev/profiles.yml +++ b/integration_tests/profiles/snowflake_dev/profiles.yml @@ -6,7 +6,7 @@ password: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_ROLE') }}" schema: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_SCHEMA') }}" - threads: 5 + threads: 4 type: snowflake user: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_USER') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_WAREHOUSE') }}" @@ -16,7 +16,7 @@ password: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_ROLE') }}" schema: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_SCHEMA') }}" - threads: 5 + threads: 4 type: snowflake user: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_USER') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_DEV_CI_WAREHOUSE') }}" From 5d3096b1024c5c7db86c4ff4dca3367d491e9a45 Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 18 Dec 2024 09:46:46 -0700 Subject: [PATCH 6/7] Add new internal CI workflow --- .../workflows/tuva_internal_dbt_v1.8.6.yml | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 .github/workflows/tuva_internal_dbt_v1.8.6.yml diff --git a/.github/workflows/tuva_internal_dbt_v1.8.6.yml b/.github/workflows/tuva_internal_dbt_v1.8.6.yml new file mode 100644 index 0000000..7284fbe --- /dev/null +++ b/.github/workflows/tuva_internal_dbt_v1.8.6.yml @@ -0,0 +1,202 @@ +name: tuva_internal_dbt_v1.8.6 +on: + workflow_dispatch: + pull_request: + branches: + - main + +env: +####### BigQuery + TUVA_BIGQUERY_TOKEN: ${{ secrets.DBT_BIGQUERY_DEMO_TOKEN }} + TUVA_BIGQUERY_PROJECT: ${{ secrets.DBT_BIGQUERY_DEMO_PROJECT }} +####### Fabric + DBT_FABRIC_CI_HOST: ${{ secrets.DBT_FABRIC_CI_HOST }} + DBT_FABRIC_SERVICE_PRINCIPAL_ID: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_ID }} + DBT_FABRIC_SERVICE_PRINCIPAL_SECRET: ${{ secrets.DBT_FABRIC_SERVICE_PRINCIPAL_SECRET }} + DBT_FABRIC_TENANT_ID: ${{ secrets.DBT_FABRIC_TENANT_ID }} + DBT_FABRIC_CI_DATABASE: ${{ secrets.DBT_FABRIC_CI_DATABASE }} + DBT_FABRIC_CI_SCHEMA: ${{ secrets.DBT_FABRIC_CI_SCHEMA }} +####### Redshift + DBT_REDSHIFT_CI_HOST: ${{ secrets.DBT_REDSHIFT_HOST }} + DBT_REDSHIFT_CI_USER: ${{ secrets.DBT_REDSHIFT_CI_USER }} + DBT_REDSHIFT_CI_PASSWORD: ${{ secrets.DBT_REDSHIFT_CI_PASSWORD }} + DBT_REDSHIFT_DEMO_DATABASE: ${{ secrets.DBT_REDSHIFT_DEMO_DATABASE }} +####### Snowflake + DBT_SNOWFLAKE_DEV_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_DEV_ACCOUNT }} + DBT_SNOWFLAKE_DEV_DATABASE: ${{ secrets.DBT_SNOWFLAKE_DEV_DATABASE }} + DBT_SNOWFLAKE_DEV_CI_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_PASSWORD }} + DBT_SNOWFLAKE_DEV_CI_ROLE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_ROLE }} + DBT_SNOWFLAKE_DEV_CI_SCHEMA: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_SCHEMA }} + DBT_SNOWFLAKE_DEV_CI_USER: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_USER }} + DBT_SNOWFLAKE_DEV_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_DEV_CI_WAREHOUSE }} + +jobs: +####### BigQuery + bigquery: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dbt-core and BigQuery adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-bigquery + + - name: Create dbt profiles.yml for BigQuery + run: | + mkdir -p ./profiles/bigquery + echo "default: + outputs: + dev: + type: bigquery + method: service-account + project: "${{ secrets.DBT_BIGQUERY_DEMO_PROJECT }}" + keyfile: ./creds.json + dataset: connector + threads: 8 + timeout_seconds: 300 + priority: interactive + target: dev" > ./profiles/bigquery/profiles.yml + working-directory: integration_tests + + - name: Create BigQuery credentials file + run: | + echo "${{ secrets.TUVA_BIGQUERY_TOKEN }}" | base64 --decode > ./creds.json + working-directory: integration_tests + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/bigquery + working-directory: integration_tests + + - name: dbt-debug + run: dbt debug --profiles-dir ./profiles/bigquery + working-directory: integration_tests + + - name: dbt-build + run: | + dbt build --full-refresh --profiles-dir ./profiles/bigquery + working-directory: integration_tests + + - name: Get the result + if: ${{ always() }} + run: echo "${{ steps.dbt-build.outputs.result }}" + shell: bash + + +####### Fabric + fabric_clinical_and_claims_enabled: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install ODBC Driver 18 for SQL Server + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + + - name: Install dbt-core and Fabric adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-fabric + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/fabric + working-directory: ci_testing + + - name: dbt-debug + run: dbt debug --profiles-dir ./profiles/fabric + working-directory: ci_testing + + - name: dbt-build + run: | + dbt build --full-refresh --profiles-dir ./profiles/fabric + working-directory: ci_testing + + - name: Get the result + if: ${{ always() }} + run: echo "${{ steps.dbt-build.outputs.result }}" + shell: bash + +####### Redshift + redshift: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dbt-core and Redshift adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-redshift + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/redshift + working-directory: integration_tests + + - name: dbt-debug + run: dbt debug --profiles-dir ./profiles/redshift + working-directory: integration_tests + + - name: dbt-build + run: | + dbt build --full-refresh --profiles-dir ./profiles/redshift + working-directory: integration_tests + + - name: Get the result + if: ${{ always() }} + run: echo "${{ steps.dbt-build.outputs.result }}" + shell: bash + +####### Snowflake + snowflake: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dbt-core and Snowflake adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-snowflake + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests + + - name: dbt-debug + run: dbt debug --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests + + - name: dbt-build + run: | + dbt build --full-refresh --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests + + - name: Get the result + if: ${{ always() }} + run: echo "${{ steps.dbt-build.outputs.result }}" + shell: bash From 762d92ef9b79f150341d2c26dae68c05f3e05200 Mon Sep 17 00:00:00 2001 From: sarah-tuva Date: Wed, 18 Dec 2024 09:47:09 -0700 Subject: [PATCH 7/7] Modify weekly workflows --- ...1.8.6_snowflake_dev_build_full_refresh.yml | 26 ++++++++++++------- ....8.6_snowflake_prod_build_full_refresh.yml | 26 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml b/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml index be40118..63d6f2b 100644 --- a/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml +++ b/.github/workflows/weekly_dbt_v1.8.6_snowflake_dev_build_full_refresh.yml @@ -22,20 +22,28 @@ jobs: steps: - uses: actions/checkout@v2 - - name: dbt-deps - uses: mwhitaker/dbt-action@v1.8.6 + - name: Set up Python + uses: actions/setup-python@v4 with: - dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" + python-version: '3.9' + + - name: Install dbt-core and Snowflake adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-snowflake + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests - name: dbt-debug - uses: mwhitaker/dbt-action@v1.8.6 - with: - dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" + run: dbt debug --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests - name: dbt-build - uses: mwhitaker/dbt-action@v1.8.6 - with: - dbt_command: "dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_dev --target dev" + run: | + dbt build --full-refresh --profiles-dir ./profiles/snowflake_dev + working-directory: integration_tests - name: Get the result if: ${{ always() }} diff --git a/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml b/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml index 21f632d..1bf4a1e 100644 --- a/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml +++ b/.github/workflows/weekly_dbt_v1.8.6_snowflake_prod_build_full_refresh.yml @@ -22,20 +22,28 @@ jobs: steps: - uses: actions/checkout@v2 - - name: dbt-deps - uses: mwhitaker/dbt-action@v1.8.6 + - name: Set up Python + uses: actions/setup-python@v4 with: - dbt_command: "dbt deps --profiles-dir ./integration_tests/profiles/snowflake_prod" + python-version: '3.9' + + - name: Install dbt-core and Snowflake adapter + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.8.6 dbt-snowflake + + - name: dbt-deps + run: dbt deps --profiles-dir ./profiles/snowflake_prod + working-directory: integration_tests - name: dbt-debug - uses: mwhitaker/dbt-action@v1.8.6 - with: - dbt_command: "dbt debug --profiles-dir ./integration_tests/profiles/snowflake_prod" + run: dbt debug --profiles-dir ./profiles/snowflake_prod + working-directory: integration_tests - name: dbt-build - uses: mwhitaker/dbt-action@v1.8.6 - with: - dbt_command: "dbt build --full-refresh --profiles-dir ./integration_tests/profiles/snowflake_prod" + run: | + dbt build --full-refresh --profiles-dir ./profiles/snowflake_prod + working-directory: integration_tests - name: Get the result if: ${{ always() }}