Feature/add semantic model (#288)

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
       - id: check-model-has-description
       - id: check-model-has-properties-file
       - id: check-model-has-all-columns
-
+        args: ['--exclude', 'active_vendor_analytics[.]sql']
   - repo: https://github.com/sqlfluff/sqlfluff
     # this is the version of sqlfluff, needs to be updated when using a new sqlfluff version (pip show sqlfluff)
     rev: 3.1.1

automate/dbt/push_dbt_artifacts.py

@@ -158,7 +158,7 @@ def delete_project_file(account_id: int, project_slug: str, filename: str):
 
     # UPLOAD FILES
 
-    filenames = ["graph.gpickle", "graph_summary.json", "partial_parse.msgpack"]
+    filenames = ["graph.gpickle", "graph_summary.json", "partial_parse.msgpack", "semantic_manifest.json","osi_document.json"]
     for filename in filenames:
         upload_env_file(account_id, project_slug, environment_slug, filename)
 

training_and_demos/dbt-api/skip_partial_parse.md

@@ -4,6 +4,8 @@ We can then push them to dbt-api on Datacoves to skip partial parse in Airflow t
 
 In order for Airflow to skip partial parse these need to be the same when the manifest and partial parse files are generated.
 
+Target name (remember, Datacoves dbt operator will use 'default' as the name if not overridden)
+
 ```yaml
       database: MY_DATABASE
       role: MY_ROLE

transform/models/L1_inlets/erp/stg_vendors.sql

@@ -27,6 +27,13 @@ archetypes as (
 
 ),
 
+archetype_count as (
+
+    select count(*) as archetype_total
+    from archetypes
+
+),
+
 generated_months as (
 
     select
@@ -46,10 +53,11 @@ generated_vendors as (
         archetypes.status,
         generated_months.month_start::date as onboarded_date
     from generated_months
+    cross join archetype_count
     join archetypes
         on archetypes.archetype_index = mod(
                 generated_months.generated_vendor_number - 1,
-                (select count(*) from archetypes)
+                archetype_count.archetype_total
             )
 
 ),

transform/models/L3_coves/accounts_payable/active_vendor_analytics.sql

@@ -0,0 +1,29 @@
+-- Snowflake semantic view DDL (tables/dimensions/metrics) is not supported by
+-- the sqlfluff snowflake dialect, so linting is disabled for this model.
+-- noqa:disable=all
+{{ config(materialized='semantic_view') }}
+
+tables (
+    active_vendor_periods as {{ ref('mart_active_vendors_by_period') }}
+)
+
+dimensions (
+    active_vendor_periods.period_grain as period_grain,
+    active_vendor_periods.period_end_date as period_end_date,
+    active_vendor_periods.vendor_id as vendor_id,
+    active_vendor_periods.vendor_name as vendor_name,
+    active_vendor_periods.category as category,
+    active_vendor_periods.recorded_status as recorded_status
+)
+
+metrics (
+    active_vendor_periods.active_vendors_weekly as count(distinct case
+        when active_vendor_periods.period_grain = 'week' then active_vendor_periods.vendor_id
+    end),
+    active_vendor_periods.active_vendors_monthly as count(distinct case
+        when active_vendor_periods.period_grain = 'month' then active_vendor_periods.vendor_id
+    end),
+    active_vendor_periods.active_vendors_quarterly as count(distinct case
+        when active_vendor_periods.period_grain = 'quarter' then active_vendor_periods.vendor_id
+    end)
+)

transform/models/L3_coves/accounts_payable/mart_active_vendors.yml

@@ -1,6 +1,9 @@
 version: 2
 
 models:
+  - name: active_vendor_analytics
+    description: Snowflake semantic view for active-vendor reporting, computed from the period-level mart at query time.
+
   - name: mart_active_vendors
     description: Accounts Payable cove for behavioral Active Vendor analysis using trailing-12-month PO-or-invoice activity.
     columns:
@@ -36,3 +39,40 @@ models:
       - name: is_active
         description: True when the vendor has at least one PO or invoice in the trailing 12 months.
         data_tests: [not_null]
+
+  - name: mart_active_vendors_by_period
+    description: Active vendors at completed weekly, monthly, and quarterly period ends using trailing-90-day PO-or-invoice activity.
+    columns:
+      - name: period_grain
+        description: Reporting period type used to anchor the trailing-90-day window.
+        data_tests:
+          - not_null
+          - accepted_values:
+              arguments:
+                values: ['week', 'month', 'quarter']
+      - name: period_end_date
+        description: Completed ISO week-, calendar month-, or calendar quarter-end date.
+        data_tests: [not_null]
+      - name: vendor_id
+        description: Vendor primary key counted distinctly by the active vendor semantic view metrics.
+        data_tests: [not_null]
+      - name: vendor_name
+        description: Vendor display name.
+        data_tests: [not_null]
+      - name: category
+        description: Vendor spend category.
+        data_tests: [not_null]
+      - name: recorded_status
+        description: Stored ERP lifecycle flag, independent from behavioral activity.
+        data_tests:
+          - not_null
+          - accepted_values:
+              arguments:
+                values: ['active', 'inactive']
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns:
+              - period_grain
+              - period_end_date
+              - vendor_id

transform/models/L3_coves/accounts_payable/mart_active_vendors_by_period.sql

@@ -0,0 +1,107 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+
+with purchase_orders as (
+
+    select
+        vendor_id,
+        po_date as activity_date
+    from {{ ref('fct_purchase_orders') }}
+
+),
+
+invoices as (
+
+    select
+        vendor_id,
+        invoice_date as activity_date
+    from {{ ref('fct_invoices') }}
+
+),
+
+activity as (
+
+    select * from purchase_orders
+    union all
+    select * from invoices
+
+),
+
+bounds as (
+
+    select
+        min(activity_date) as first_activity_date,
+        max(activity_date) as latest_activity_date
+    from activity
+
+),
+
+calendar_days as (
+
+    select dateadd(day, seq4(), bounds.first_activity_date) as calendar_date
+    from bounds,
+        table(generator(rowcount => 10000))
+    where dateadd(day, seq4(), bounds.first_activity_date) <= bounds.latest_activity_date
+
+),
+
+period_ends as (
+
+    select distinct
+        'week' as period_grain,
+        dateadd(day, 7 - dayofweekiso(calendar_date), calendar_date) as period_end_date
+    from calendar_days
+
+    union all
+
+    select distinct
+        'month' as period_grain,
+        last_day(calendar_date, 'month') as period_end_date
+    from calendar_days
+
+    union all
+
+    select distinct
+        'quarter' as period_grain,
+        last_day(calendar_date, 'quarter') as period_end_date
+    from calendar_days
+
+),
+
+completed_period_ends as (
+
+    select
+        period_ends.period_grain,
+        period_ends.period_end_date
+    from period_ends
+    cross join bounds
+    where period_ends.period_end_date <= bounds.latest_activity_date
+
+),
+
+active_vendor_periods as (
+
+    select distinct
+        completed_period_ends.period_grain,
+        completed_period_ends.period_end_date,
+        activity.vendor_id
+    from completed_period_ends
+    inner join activity
+        on activity.activity_date > dateadd(day, -90, completed_period_ends.period_end_date)
+            and activity.activity_date <= completed_period_ends.period_end_date
+
+)
+
+select
+    active_vendor_periods.period_grain,
+    active_vendor_periods.period_end_date,
+    active_vendor_periods.vendor_id,
+    vendors.vendor_name,
+    vendors.category,
+    vendors.status as recorded_status
+from active_vendor_periods
+inner join {{ ref('dim_vendors') }} as vendors
+    on active_vendor_periods.vendor_id = vendors.vendor_id

transform/package-lock.yml

@@ -11,7 +11,10 @@ packages:
   - name: dbt_external_tables
     package: dbt-labs/dbt_external_tables
     version: 0.12.1
+  - name: dbt_semantic_view
+    package: Snowflake-Labs/dbt_semantic_view
+    version: 1.0.6
   - name: dbt_date
     package: godatadriven/dbt_date
-    version: 0.17.1
-sha1_hash: c22f502b6cd00019053b254ea09eba149cdd1379
+    version: 0.21.0
+sha1_hash: 0eaeb8ea5ebfc5f272342bae3279837f85985981

transform/packages.yml

@@ -9,6 +9,8 @@ packages:
   - package: dbt-labs/dbt_external_tables
     version: 0.12.1
     # <see https://github.com/dbt-labs/dbt-external-tables/releases/tag/0.7.1> for the latest version tag
+  - package: Snowflake-Labs/dbt_semantic_view
+    version: 1.0.6
   # - package: data-mie/dbt_profiler
   #   version: 0.8.4
   # - package: dbt-labs/audit_helper

transform/tests/L3_coves/accounts_payable/active_vendor_period_has_recent_activity.sql

@@ -0,0 +1,30 @@
+with activity as (
+
+    select
+        vendor_id,
+        po_date as activity_date
+    from {{ ref('fct_purchase_orders') }}
+
+    union all
+
+    select
+        vendor_id,
+        invoice_date as activity_date
+    from {{ ref('fct_invoices') }}
+
+)
+
+select
+    active_vendor_periods.period_grain,
+    active_vendor_periods.period_end_date,
+    active_vendor_periods.vendor_id
+from {{ ref('mart_active_vendors_by_period') }} as active_vendor_periods
+left join activity
+    on active_vendor_periods.vendor_id = activity.vendor_id
+        and activity.activity_date > dateadd(day, -90, active_vendor_periods.period_end_date)
+        and activity.activity_date <= active_vendor_periods.period_end_date
+group by
+    active_vendor_periods.period_grain,
+    active_vendor_periods.period_end_date,
+    active_vendor_periods.vendor_id
+having count(activity.activity_date) = 0

transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql

@@ -2,22 +2,35 @@
 
 with reference_dates as (
 
-    select to_date('2026-06-01') as reference_date, 32 as expected_total, 17 as expected_active
+    select
+        to_date('2026-06-01') as reference_date,
+        32 as expected_total,
+        17 as expected_active
     union all
-    select to_date('2026-07-01') as reference_date, 33 as expected_total, 24 as expected_active
+    select
+        to_date('2026-07-01') as reference_date,
+        33 as expected_total,
+        24 as expected_active
     union all
-    select to_date('2026-08-01') as reference_date, 34 as expected_total, 25 as expected_active
+    select
+        to_date('2026-08-01') as reference_date,
+        34 as expected_total,
+        25 as expected_active
 
 ),
 
 activity as (
 
-    select vendor_id, po_date as activity_date
+    select
+        vendor_id,
+        po_date as activity_date
     from {{ ref('fct_purchase_orders') }}
 
     union all
 
-    select vendor_id, invoice_date as activity_date
+    select
+        vendor_id,
+        invoice_date as activity_date
     from {{ ref('fct_invoices') }}
 
 ),
@@ -33,8 +46,8 @@ actual_counts as (
         on dim_vendors.onboarded_date <= reference_dates.reference_date
     left join activity
         on dim_vendors.vendor_id = activity.vendor_id
-        and activity.activity_date > dateadd(month, -12, reference_dates.reference_date)
-        and activity.activity_date <= reference_dates.reference_date
+            and activity.activity_date > dateadd(month, -12, reference_dates.reference_date)
+            and activity.activity_date <= reference_dates.reference_date
     group by reference_dates.reference_date
 
 )