diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1872e431..c6e1af5e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - id: check-model-has-description - id: check-model-has-properties-file - id: check-model-has-all-columns - + args: ['--exclude', 'active_vendor_analytics[.]sql'] - repo: https://github.com/sqlfluff/sqlfluff # this is the version of sqlfluff, needs to be updated when using a new sqlfluff version (pip show sqlfluff) rev: 3.1.1 diff --git a/automate/dbt/push_dbt_artifacts.py b/automate/dbt/push_dbt_artifacts.py index 1fc02fcc..d6758b22 100755 --- a/automate/dbt/push_dbt_artifacts.py +++ b/automate/dbt/push_dbt_artifacts.py @@ -158,7 +158,7 @@ def delete_project_file(account_id: int, project_slug: str, filename: str): # UPLOAD FILES - filenames = ["graph.gpickle", "graph_summary.json", "partial_parse.msgpack"] + filenames = ["graph.gpickle", "graph_summary.json", "partial_parse.msgpack", "semantic_manifest.json","osi_document.json"] for filename in filenames: upload_env_file(account_id, project_slug, environment_slug, filename) diff --git a/training_and_demos/dbt-api/skip_partial_parse.md b/training_and_demos/dbt-api/skip_partial_parse.md index 5c4bba6e..9ebd07f0 100644 --- a/training_and_demos/dbt-api/skip_partial_parse.md +++ b/training_and_demos/dbt-api/skip_partial_parse.md @@ -4,6 +4,8 @@ We can then push them to dbt-api on Datacoves to skip partial parse in Airflow t In order for Airflow to skip partial parse these need to be the same when the manifest and partial parse files are generated. +Target name (remember, Datacoves dbt operator will use 'default' as the name if not overridden) + ```yaml database: MY_DATABASE role: MY_ROLE diff --git a/transform/models/L1_inlets/erp/stg_vendors.sql b/transform/models/L1_inlets/erp/stg_vendors.sql index c45bfb9f..314b42f7 100644 --- a/transform/models/L1_inlets/erp/stg_vendors.sql +++ b/transform/models/L1_inlets/erp/stg_vendors.sql @@ -27,6 +27,13 @@ archetypes as ( ), +archetype_count as ( + + select count(*) as archetype_total + from archetypes + +), + generated_months as ( select @@ -46,10 +53,11 @@ generated_vendors as ( archetypes.status, generated_months.month_start::date as onboarded_date from generated_months + cross join archetype_count join archetypes on archetypes.archetype_index = mod( generated_months.generated_vendor_number - 1, - (select count(*) from archetypes) + archetype_count.archetype_total ) ), diff --git a/transform/models/L3_coves/accounts_payable/active_vendor_analytics.sql b/transform/models/L3_coves/accounts_payable/active_vendor_analytics.sql new file mode 100644 index 00000000..57c4ee0a --- /dev/null +++ b/transform/models/L3_coves/accounts_payable/active_vendor_analytics.sql @@ -0,0 +1,29 @@ +-- Snowflake semantic view DDL (tables/dimensions/metrics) is not supported by +-- the sqlfluff snowflake dialect, so linting is disabled for this model. +-- noqa:disable=all +{{ config(materialized='semantic_view') }} + +tables ( + active_vendor_periods as {{ ref('mart_active_vendors_by_period') }} +) + +dimensions ( + active_vendor_periods.period_grain as period_grain, + active_vendor_periods.period_end_date as period_end_date, + active_vendor_periods.vendor_id as vendor_id, + active_vendor_periods.vendor_name as vendor_name, + active_vendor_periods.category as category, + active_vendor_periods.recorded_status as recorded_status +) + +metrics ( + active_vendor_periods.active_vendors_weekly as count(distinct case + when active_vendor_periods.period_grain = 'week' then active_vendor_periods.vendor_id + end), + active_vendor_periods.active_vendors_monthly as count(distinct case + when active_vendor_periods.period_grain = 'month' then active_vendor_periods.vendor_id + end), + active_vendor_periods.active_vendors_quarterly as count(distinct case + when active_vendor_periods.period_grain = 'quarter' then active_vendor_periods.vendor_id + end) +) diff --git a/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml b/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml index c26b73e1..f5516ffe 100644 --- a/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml +++ b/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml @@ -1,6 +1,9 @@ version: 2 models: + - name: active_vendor_analytics + description: Snowflake semantic view for active-vendor reporting, computed from the period-level mart at query time. + - name: mart_active_vendors description: Accounts Payable cove for behavioral Active Vendor analysis using trailing-12-month PO-or-invoice activity. columns: @@ -36,3 +39,40 @@ models: - name: is_active description: True when the vendor has at least one PO or invoice in the trailing 12 months. data_tests: [not_null] + + - name: mart_active_vendors_by_period + description: Active vendors at completed weekly, monthly, and quarterly period ends using trailing-90-day PO-or-invoice activity. + columns: + - name: period_grain + description: Reporting period type used to anchor the trailing-90-day window. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['week', 'month', 'quarter'] + - name: period_end_date + description: Completed ISO week-, calendar month-, or calendar quarter-end date. + data_tests: [not_null] + - name: vendor_id + description: Vendor primary key counted distinctly by the active vendor semantic view metrics. + data_tests: [not_null] + - name: vendor_name + description: Vendor display name. + data_tests: [not_null] + - name: category + description: Vendor spend category. + data_tests: [not_null] + - name: recorded_status + description: Stored ERP lifecycle flag, independent from behavioral activity. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: + - period_grain + - period_end_date + - vendor_id diff --git a/transform/models/L3_coves/accounts_payable/mart_active_vendors_by_period.sql b/transform/models/L3_coves/accounts_payable/mart_active_vendors_by_period.sql new file mode 100644 index 00000000..4759322e --- /dev/null +++ b/transform/models/L3_coves/accounts_payable/mart_active_vendors_by_period.sql @@ -0,0 +1,107 @@ +{{ + config( + materialized='table' + ) +}} + +with purchase_orders as ( + + select + vendor_id, + po_date as activity_date + from {{ ref('fct_purchase_orders') }} + +), + +invoices as ( + + select + vendor_id, + invoice_date as activity_date + from {{ ref('fct_invoices') }} + +), + +activity as ( + + select * from purchase_orders + union all + select * from invoices + +), + +bounds as ( + + select + min(activity_date) as first_activity_date, + max(activity_date) as latest_activity_date + from activity + +), + +calendar_days as ( + + select dateadd(day, seq4(), bounds.first_activity_date) as calendar_date + from bounds, + table(generator(rowcount => 10000)) + where dateadd(day, seq4(), bounds.first_activity_date) <= bounds.latest_activity_date + +), + +period_ends as ( + + select distinct + 'week' as period_grain, + dateadd(day, 7 - dayofweekiso(calendar_date), calendar_date) as period_end_date + from calendar_days + + union all + + select distinct + 'month' as period_grain, + last_day(calendar_date, 'month') as period_end_date + from calendar_days + + union all + + select distinct + 'quarter' as period_grain, + last_day(calendar_date, 'quarter') as period_end_date + from calendar_days + +), + +completed_period_ends as ( + + select + period_ends.period_grain, + period_ends.period_end_date + from period_ends + cross join bounds + where period_ends.period_end_date <= bounds.latest_activity_date + +), + +active_vendor_periods as ( + + select distinct + completed_period_ends.period_grain, + completed_period_ends.period_end_date, + activity.vendor_id + from completed_period_ends + inner join activity + on activity.activity_date > dateadd(day, -90, completed_period_ends.period_end_date) + and activity.activity_date <= completed_period_ends.period_end_date + +) + +select + active_vendor_periods.period_grain, + active_vendor_periods.period_end_date, + active_vendor_periods.vendor_id, + vendors.vendor_name, + vendors.category, + vendors.status as recorded_status +from active_vendor_periods +inner join {{ ref('dim_vendors') }} as vendors + on active_vendor_periods.vendor_id = vendors.vendor_id diff --git a/transform/package-lock.yml b/transform/package-lock.yml index 70e2163c..ac814876 100644 --- a/transform/package-lock.yml +++ b/transform/package-lock.yml @@ -11,7 +11,10 @@ packages: - name: dbt_external_tables package: dbt-labs/dbt_external_tables version: 0.12.1 + - name: dbt_semantic_view + package: Snowflake-Labs/dbt_semantic_view + version: 1.0.6 - name: dbt_date package: godatadriven/dbt_date - version: 0.17.1 -sha1_hash: c22f502b6cd00019053b254ea09eba149cdd1379 + version: 0.21.0 +sha1_hash: 0eaeb8ea5ebfc5f272342bae3279837f85985981 diff --git a/transform/packages.yml b/transform/packages.yml index a1aabbc0..8b295b71 100644 --- a/transform/packages.yml +++ b/transform/packages.yml @@ -9,6 +9,8 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.12.1 # for the latest version tag + - package: Snowflake-Labs/dbt_semantic_view + version: 1.0.6 # - package: data-mie/dbt_profiler # version: 0.8.4 # - package: dbt-labs/audit_helper diff --git a/transform/tests/L3_coves/accounts_payable/active_vendor_period_has_recent_activity.sql b/transform/tests/L3_coves/accounts_payable/active_vendor_period_has_recent_activity.sql new file mode 100644 index 00000000..69ebb8ea --- /dev/null +++ b/transform/tests/L3_coves/accounts_payable/active_vendor_period_has_recent_activity.sql @@ -0,0 +1,30 @@ +with activity as ( + + select + vendor_id, + po_date as activity_date + from {{ ref('fct_purchase_orders') }} + + union all + + select + vendor_id, + invoice_date as activity_date + from {{ ref('fct_invoices') }} + +) + +select + active_vendor_periods.period_grain, + active_vendor_periods.period_end_date, + active_vendor_periods.vendor_id +from {{ ref('mart_active_vendors_by_period') }} as active_vendor_periods +left join activity + on active_vendor_periods.vendor_id = activity.vendor_id + and activity.activity_date > dateadd(day, -90, active_vendor_periods.period_end_date) + and activity.activity_date <= active_vendor_periods.period_end_date +group by + active_vendor_periods.period_grain, + active_vendor_periods.period_end_date, + active_vendor_periods.vendor_id +having count(activity.activity_date) = 0 diff --git a/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql b/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql index 90e56cfa..06dbcc7e 100644 --- a/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql +++ b/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql @@ -2,22 +2,35 @@ with reference_dates as ( - select to_date('2026-06-01') as reference_date, 32 as expected_total, 17 as expected_active + select + to_date('2026-06-01') as reference_date, + 32 as expected_total, + 17 as expected_active union all - select to_date('2026-07-01') as reference_date, 33 as expected_total, 24 as expected_active + select + to_date('2026-07-01') as reference_date, + 33 as expected_total, + 24 as expected_active union all - select to_date('2026-08-01') as reference_date, 34 as expected_total, 25 as expected_active + select + to_date('2026-08-01') as reference_date, + 34 as expected_total, + 25 as expected_active ), activity as ( - select vendor_id, po_date as activity_date + select + vendor_id, + po_date as activity_date from {{ ref('fct_purchase_orders') }} union all - select vendor_id, invoice_date as activity_date + select + vendor_id, + invoice_date as activity_date from {{ ref('fct_invoices') }} ), @@ -33,8 +46,8 @@ actual_counts as ( on dim_vendors.onboarded_date <= reference_dates.reference_date left join activity on dim_vendors.vendor_id = activity.vendor_id - and activity.activity_date > dateadd(month, -12, reference_dates.reference_date) - and activity.activity_date <= reference_dates.reference_date + and activity.activity_date > dateadd(month, -12, reference_dates.reference_date) + and activity.activity_date <= reference_dates.reference_date group by reference_dates.reference_date )