From 06868457eb753580e6f3b3269d63749afea670b0 Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 24 Aug 2026 09:13:29 +0200 Subject: [PATCH] perf: precompute org page contributors leaderboard Signed-off-by: anilb --- .../org_page_contributors_copy_ds.datasource | 16 +++ .../tinybird/pipes/org_page_contributors.pipe | 108 ++++++++++++------ .../org_page_contributors_copy_pipe.pipe | 17 +++ 3 files changed, 106 insertions(+), 35 deletions(-) create mode 100644 services/libs/tinybird/datasources/org_page_contributors_copy_ds.datasource create mode 100644 services/libs/tinybird/pipes/org_page_contributors_copy_pipe.pipe diff --git a/services/libs/tinybird/datasources/org_page_contributors_copy_ds.datasource b/services/libs/tinybird/datasources/org_page_contributors_copy_ds.datasource new file mode 100644 index 0000000000..fcfbc9c277 --- /dev/null +++ b/services/libs/tinybird/datasources/org_page_contributors_copy_ds.datasource @@ -0,0 +1,16 @@ +DESCRIPTION > + Precomputed all-time contribution counts per (organizationId, memberId) for the org + page contributors leaderboard. Rebuilt nightly by org_page_contributors_copy_pipe. + One row per (organizationId, memberId) pair (~2M rows total). Used by + org_page_contributors.pipe for cheap request-time lookups instead of scanning the + full activityRelations cleaned bucket union (~744M rows) per request. + +SCHEMA > + `organizationId` String, + `memberId` String, + `contributionCount` UInt64, + `computedAt` DateTime + +ENGINE ReplacingMergeTree +ENGINE_SORTING_KEY organizationId, memberId +ENGINE_VER computedAt diff --git a/services/libs/tinybird/pipes/org_page_contributors.pipe b/services/libs/tinybird/pipes/org_page_contributors.pipe index 672816bbcc..f023e93e0a 100644 --- a/services/libs/tinybird/pipes/org_page_contributors.pipe +++ b/services/libs/tinybird/pipes/org_page_contributors.pipe @@ -1,6 +1,10 @@ DESCRIPTION > Top contributors for a given organization leaderboard. Returns members sorted by contribution count within the specified date range. + When no startDate/endDate is given (the dominant traffic shape), reads precomputed + per-(organizationId, memberId) counts from org_page_contributors_copy_ds (keyed + lookup) instead of scanning the full activityRelations cleaned bucket union + (~744M rows) per request. Date-filtered requests fall back to the full scan. TAGS "Organization page" @@ -14,49 +18,83 @@ SQL > NODE org_page_contributors_activity_aggregates SQL > % + {% set use_precomputed = 1 %} + {% if defined(startDate) %} {% set use_precomputed = 0 %} {% end %} + {% if defined(endDate) %} {% set use_precomputed = 0 %} {% end %} {% if Boolean(count, false) %} - SELECT count(distinct memberId) - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE - organizationId = (SELECT id FROM org_slug_lookup) - {% if defined(startDate) %} - AND timestamp - >= {{ DateTime(startDate, description="Filter activity timestamp after") }} - {% end %} - {% if defined(endDate) %} - AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }} - {% end %} + {% if use_precomputed == 1 %} + SELECT count() + FROM org_page_contributors_copy_ds + WHERE organizationId = (SELECT id FROM org_slug_lookup) + {% else %} + SELECT count(distinct memberId) + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE + organizationId = (SELECT id FROM org_slug_lookup) + {% if defined(startDate) %} + AND timestamp + >= {{ DateTime(startDate, description="Filter activity timestamp after") }} + {% end %} + {% if defined(endDate) %} + AND timestamp + < {{ DateTime(endDate, description="Filter activity timestamp before") }} + {% end %} + {% end %} {% else %} - SELECT - memberId, - count() as "contributionCount", - ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) as "contributionPercentage" - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE - organizationId = (SELECT id FROM org_slug_lookup) - {% if defined(startDate) %} - AND timestamp - >= {{ DateTime(startDate, description="Filter activity timestamp after") }} - {% end %} - {% if defined(endDate) %} - AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }} - {% end %} - GROUP BY memberId - ORDER BY contributionCount DESC, memberId DESC - LIMIT {{ Int32(limit, 10) }} - OFFSET {{ Int32(offset, 0) }} + {% if use_precomputed == 1 %} + SELECT + memberId, + contributionCount, + ROUND( + contributionCount * 100.0 / SUM(contributionCount) OVER (), 2 + ) as "contributionPercentage" + FROM org_page_contributors_copy_ds + WHERE organizationId = (SELECT id FROM org_slug_lookup) + ORDER BY contributionCount DESC, memberId DESC + LIMIT {{ Int32(limit, 10) }} + OFFSET {{ Int32(offset, 0) }} + {% else %} + SELECT + memberId, + count() as "contributionCount", + ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) as "contributionPercentage" + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE + organizationId = (SELECT id FROM org_slug_lookup) + {% if defined(startDate) %} + AND timestamp + >= {{ DateTime(startDate, description="Filter activity timestamp after") }} + {% end %} + {% if defined(endDate) %} + AND timestamp + < {{ DateTime(endDate, description="Filter activity timestamp before") }} + {% end %} + GROUP BY memberId + ORDER BY contributionCount DESC, memberId DESC + LIMIT {{ Int32(limit, 10) }} + OFFSET {{ Int32(offset, 0) }} + {% end %} {% end %} NODE org_page_contributors_leaderboard SQL > % + {% set use_precomputed = 1 %} + {% if defined(startDate) %} {% set use_precomputed = 0 %} {% end %} + {% if defined(endDate) %} {% set use_precomputed = 0 %} {% end %} {% if Boolean(count, false) %} - SELECT count(distinct memberId) as count - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE - organizationId = (SELECT id FROM org_slug_lookup) - {% if defined(startDate) %} AND timestamp >= {{ DateTime(startDate) }} {% end %} - {% if defined(endDate) %} AND timestamp < {{ DateTime(endDate) }} {% end %} + {% if use_precomputed == 1 %} + SELECT count() as count + FROM org_page_contributors_copy_ds + WHERE organizationId = (SELECT id FROM org_slug_lookup) + {% else %} + SELECT count(distinct memberId) as count + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE + organizationId = (SELECT id FROM org_slug_lookup) + {% if defined(startDate) %} AND timestamp >= {{ DateTime(startDate) }} {% end %} + {% if defined(endDate) %} AND timestamp < {{ DateTime(endDate) }} {% end %} + {% end %} {% else %} SELECT m.id, diff --git a/services/libs/tinybird/pipes/org_page_contributors_copy_pipe.pipe b/services/libs/tinybird/pipes/org_page_contributors_copy_pipe.pipe new file mode 100644 index 0000000000..87135ba32f --- /dev/null +++ b/services/libs/tinybird/pipes/org_page_contributors_copy_pipe.pipe @@ -0,0 +1,17 @@ +DESCRIPTION > + Nightly copy pipe that precomputes all-time contribution counts per + (organizationId, memberId) for the org page contributors leaderboard. + +TAGS "Organization page" + +NODE org_page_contributors_copy_pipe_data +SQL > + SELECT organizationId, memberId, count() AS contributionCount, now() AS computedAt + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE organizationId != '' + GROUP BY organizationId, memberId + +TYPE COPY +TARGET_DATASOURCE org_page_contributors_copy_ds +COPY_MODE replace +COPY_SCHEDULE 50 3 * * *