diff --git a/README.md b/README.md index c73efe0..98d629a 100644 --- a/README.md +++ b/README.md @@ -73,3 +73,15 @@ Set `BETTER_AUTH_SECRET` in `.env`. Either set `DATABASE_URL` (your Postgres; no --- + +## Domain checks and snapshots + +Domain detail pages support an on-demand public check. The refresh endpoint runs the existing DNS, TLS, registration and host enrichment pipeline and stores an append-only snapshot for troubleshooting. + +- `POST /api/domains/:id/checks` runs a refresh for a domain owned by the signed-in user. +- `GET /api/domains/:id/checks?limit=20` returns recent snapshots. The limit defaults to 20 and is capped at 100. +- Refreshes are session-authenticated, rate-limited to one check per domain per minute and 24 checks per user per minute (per application process). +- Snapshots are retained in `domain_check_runs`; deleting a domain cascades to its snapshots. The latest snapshot is also reflected in the existing `domains.enrichment` field for compatibility. +- Run `bun run db:migrate` before starting an upgraded deployment. + +The first version checks the domain’s DNS records, TLS certificate and existing registration/host enrichment. It does not run a separate HTTP status probe yet; scheduled portfolio-wide checks and webhook notifications are tracked separately. diff --git a/drizzle/0003_domain_check_runs.sql b/drizzle/0003_domain_check_runs.sql new file mode 100644 index 0000000..138fa9c --- /dev/null +++ b/drizzle/0003_domain_check_runs.sql @@ -0,0 +1,12 @@ +CREATE TABLE "domain_check_runs" ( + "id" text PRIMARY KEY NOT NULL, + "domainId" text NOT NULL, + "checkedAt" timestamp (3) DEFAULT now() NOT NULL, + "result" jsonb NOT NULL, + "error" text +); +--> statement-breakpoint +ALTER TABLE "domain_check_runs" ADD CONSTRAINT "domain_check_runs_domainId_domains_id_fk" + FOREIGN KEY ("domainId") REFERENCES "public"."domains"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +CREATE INDEX "domain_check_runs_domainId_checkedAt_idx" ON "domain_check_runs" USING btree ("domainId", "checkedAt"); diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index d8a5c8a..ae42a0c 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -2,26 +2,9 @@ "version": "7", "dialect": "postgresql", "entries": [ - { - "idx": 0, - "version": "7", - "when": 1774095492703, - "tag": "0000_spooky_odin", - "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1774437795200, - "tag": "0001_strange_jasper_sitwell", - "breakpoints": true - }, - { - "idx": 2, - "version": "7", - "when": 1775055276544, - "tag": "0002_outstanding_bishop", - "breakpoints": true - } + { "idx": 0, "version": "7", "when": 1774095492703, "tag": "0000_spooky_odin", "breakpoints": true }, + { "idx": 1, "version": "7", "when": 1774437795200, "tag": "0001_strange_jasper_sitwell", "breakpoints": true }, + { "idx": 2, "version": "7", "when": 1775055276544, "tag": "0002_outstanding_bishop", "breakpoints": true }, + { "idx": 3, "version": "7", "when": 1793550000000, "tag": "0003_domain_check_runs", "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/frontend/src/pages/domains/DomainDetail.tsx b/frontend/src/pages/domains/DomainDetail.tsx index 5dec03b..6ee7262 100644 --- a/frontend/src/pages/domains/DomainDetail.tsx +++ b/frontend/src/pages/domains/DomainDetail.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { Link, useNavigate, useParams } from "react-router-dom"; import { ArrowsClockwise, @@ -16,7 +16,7 @@ import { Trash, Warning, } from "@phosphor-icons/react"; -import { deleteDomain, refreshDomainPublic, useDomain } from "../hooks/useDomains"; +import { deleteDomain, fetchDomainCheckHistory, runDomainCheck, type DomainCheckRun, useDomain } from "../hooks/useDomains"; import { effectiveExpiry, effectiveRegistrar, @@ -134,6 +134,9 @@ export function DomainDetail() { const [deleting, setDeleting] = useState(false); const [refreshError, setRefreshError] = useState(null); const [refreshing, setRefreshing] = useState(false); + const [checkRuns, setCheckRuns] = useState([]); + const [historyError, setHistoryError] = useState(null); + const [historyLoading, setHistoryLoading] = useState(false); const deleteDialogRef = useRef(null); const enrichment = domain?.enrichment ?? null; @@ -147,6 +150,23 @@ export function DomainDetail() { const historyEvents = useMemo(() => sortRdapEventsDesc(rdap?.events ?? []), [rdap?.events]); + useEffect(() => { + if (!id) return; + let cancelled = false; + setHistoryLoading(true); + void fetchDomainCheckHistory(id, 10).then((response) => { + if (cancelled) return; + setHistoryLoading(false); + if (!response.ok) { + setHistoryError(response.error); + return; + } + setHistoryError(null); + setCheckRuns(response.runs); + }); + return () => { cancelled = true; }; + }, [id]); + const dataSourcesLine = useMemo(() => { const m = enrichment?.enrichmentMetadata; if (!m) return null; @@ -161,17 +181,31 @@ export function DomainDetail() { [domain?.enrichedAt, enrichment?.fetchedAt] ); + + async function loadCheckHistory() { + if (!id) return; + setHistoryLoading(true); + const response = await fetchDomainCheckHistory(id, 10); + setHistoryLoading(false); + if (!response.ok) { + setHistoryError(response.error); + return; + } + setHistoryError(null); + setCheckRuns(response.runs); + } + async function handleRefreshPublic() { if (!id) return; setRefreshError(null); setRefreshing(true); - const result = await refreshDomainPublic(id); + const result = await runDomainCheck(id); setRefreshing(false); if (!result.ok) { setRefreshError(result.error); return; } - await reload(); + await Promise.all([reload(), loadCheckHistory()]); } async function runDelete() { @@ -223,6 +257,8 @@ export function DomainDetail() { const snapshotTime = domain.enrichedAt ?? enrichment?.fetchedAt; + + return (
+
+
+

Recent checks

+ +
+ {historyError ?

{historyError}

: null} + {!historyError && !checkRuns.length && !historyLoading ? ( +

No saved check runs yet. Choose “Refresh data” to create the first snapshot.

+ ) : null} + {checkRuns.length ? ( +
    + {checkRuns.map((run) => ( +
  1. + + + {run.error || run.result.errors?.length ? "Completed with warnings" : "Check completed"} + + +
    + View snapshot + 
    +                        {JSON.stringify(run.result, null, 2)}
+
    +
    +
    2. + ))} +
+ ) : null} +
diff --git a/frontend/src/pages/hooks/useDomains.ts b/frontend/src/pages/hooks/useDomains.ts index 54f25c7..95602ef 100644 --- a/frontend/src/pages/hooks/useDomains.ts +++ b/frontend/src/pages/hooks/useDomains.ts @@ -61,6 +61,39 @@ export async function deleteDomain( return { ok: true }; } +export type DomainCheckRun = { + id: string; + checkedAt: string; + result: DomainEnrichment; + error: string | null; +}; + +type CheckHistoryResponse = { runs: DomainCheckRun[] }; +type RefreshCheckResponse = { run: DomainCheckRun }; + +export async function fetchDomainCheckHistory(id: string, limit = 10): Promise< + { ok: true; runs: DomainCheckRun[] } | { ok: false; error: string } +> { + const res = await httpGet(`/api/domains/${encodeURIComponent(id)}/checks?limit=${limit}`); + if (!res.ok) return { ok: false, error: await readError(res) }; + const data = (await res.json()) as CheckHistoryResponse; + return { ok: true, runs: Array.isArray(data.runs) ? data.runs : [] }; +} + +export async function runDomainCheck(id: string): Promise< + { ok: true; run: DomainCheckRun } | { ok: false; error: string } +> { + try { + const res = await httpPost(`/api/domains/${encodeURIComponent(id)}/checks`, { json: {} }); + if (!res.ok) return { ok: false, error: await readError(res) }; + const data = (await res.json()) as RefreshCheckResponse; + if (!data.run) return { ok: false, error: "Invalid check response" }; + return { ok: true, run: data.run }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : "Request failed" }; + } +} + type LookupResponse = { enrichment: DomainEnrichment }; export async function fetchDomainLookup(hostname: string): Promise< diff --git a/src/controllers/domain.controller.ts b/src/controllers/domain.controller.ts index 06daa9f..5062456 100644 --- a/src/controllers/domain.controller.ts +++ b/src/controllers/domain.controller.ts @@ -10,6 +10,7 @@ import { updateDomainForUserRequest, } from "../services/domain.service"; import { exportDomainsPortfolioResponse, importDomainsPortfolio } from "../services/domainPortfolio.service"; +import { listDomainCheckRunsForUser, refreshDomainCheckForUser } from "../services/domainCheck.service"; const domainOpenapi = { tags: ["domains"], @@ -91,6 +92,32 @@ export const domainRoutes = new Elysia().group("/api/domains", (app) => }, } ) + .post( + "/:id/checks", + async ({ request, params }) => + toResponse(await refreshDomainCheckForUser(await getUserIdFromRequest(request), params.id)), + { + detail: { + ...domainOpenapi, + summary: "Refresh domain checks", + description: "Run DNS, TLS, registration and host checks; append a snapshot to history. Limited to one refresh per domain per minute.", + }, + } + ) + .get( + "/:id/checks", + async ({ request, params }) => { + const limit = new URL(request.url).searchParams.get("limit"); + return toResponse(await listDomainCheckRunsForUser(await getUserIdFromRequest(request), params.id, limit)); + }, + { + detail: { + ...domainOpenapi, + summary: "List domain check history", + description: "Returns recent append-only check snapshots (default 20, maximum 100).", + }, + } + ) .get( "/:id", async ({ request, params }) => diff --git a/src/db/schema.ts b/src/db/schema.ts index 878b1a8..5a6e474 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -6,6 +6,7 @@ import { text, timestamp, unique, + index, } from "drizzle-orm/pg-core"; import type { DomainEnrichment } from "../domains/enrichment/types"; @@ -169,6 +170,15 @@ export const infrastructureServers = pgTable("infrastructure_servers", { .defaultNow(), }); +/** Append-only results for explicit domain checks; deleting a domain deletes its history. */ +export const domainCheckRuns = pgTable("domain_check_runs", { + id: text("id").primaryKey(), + domainId: text("domainId").notNull().references(() => domains.id, { onDelete: "cascade" }), + checkedAt: timestamp("checkedAt", { mode: "date", precision: 3 }).notNull().defaultNow(), + result: jsonb("result").$type().notNull(), + error: text("error"), +}, (t) => [index("domain_check_runs_domainId_checkedAt_idx").on(t.domainId, t.checkedAt)]); + export const authSchema = { user, session, diff --git a/src/domains/enrichment/ratelimit.test.ts b/src/domains/enrichment/ratelimit.test.ts new file mode 100644 index 0000000..f064ad9 --- /dev/null +++ b/src/domains/enrichment/ratelimit.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from "bun:test"; +import { acquireDomainCheck, rateLimitDomainCheck, rateLimitDomainLookup } from "./ratelimit"; + +describe("domain check rate limits", () => { + test("limits one explicit refresh per domain per minute", () => { + const user = crypto.randomUUID(); + const domain = crypto.randomUUID(); + expect(rateLimitDomainCheck(user, domain).ok).toBe(true); + const blocked = rateLimitDomainCheck(user, domain); + expect(blocked.ok).toBe(false); + if (!blocked.ok) expect(blocked.retryAfterSec).toBeGreaterThan(0); + }); + + test("allows a different domain for the same user", () => { + const user = crypto.randomUUID(); + expect(rateLimitDomainCheck(user, crypto.randomUUID()).ok).toBe(true); + expect(rateLimitDomainCheck(user, crypto.randomUUID()).ok).toBe(true); + }); + + test("keeps preview lookup allowance independent from explicit checks", () => { + const user = crypto.randomUUID(); + expect(rateLimitDomainCheck(user, crypto.randomUUID()).ok).toBe(true); + expect(rateLimitDomainLookup(user).ok).toBe(true); + }); + + test("prevents overlapping checks for the same domain and releases the lock", () => { + const user = crypto.randomUUID(); + const domain = crypto.randomUUID(); + const release = acquireDomainCheck(user, domain); + expect(typeof release).toBe("function"); + expect(acquireDomainCheck(user, domain)).toBeNull(); + release?.(); + const reacquired = acquireDomainCheck(user, domain); + expect(typeof reacquired).toBe("function"); + reacquired?.(); + }); +}); diff --git a/src/domains/enrichment/ratelimit.ts b/src/domains/enrichment/ratelimit.ts index 3b5de62..3c0d5be 100644 --- a/src/domains/enrichment/ratelimit.ts +++ b/src/domains/enrichment/ratelimit.ts @@ -1,18 +1,42 @@ const WINDOW_MS = 60_000; const MAX_REQUESTS = 24; +const DOMAIN_WINDOW_MS = 60_000; +const MAX_DOMAIN_REQUESTS = 1; -const hits = new Map(); +const lookupHits = new Map(); +const checkHits = new Map(); +const domainHits = new Map(); +const inFlightChecks = new Set(); -export function rateLimitDomainLookup(userId: string): { ok: true } | { ok: false; retryAfterSec: number } { +function consume(map: Map, key: string, max: number, windowMs: number): + { ok: true } | { ok: false; retryAfterSec: number } { const now = Date.now(); - const prev = hits.get(userId) ?? []; - const fresh = prev.filter((t) => now - t < WINDOW_MS); - if (fresh.length >= MAX_REQUESTS) { - const oldest = Math.min(...fresh); - const retryAfterSec = Math.ceil((WINDOW_MS - (now - oldest)) / 1000); + const fresh = (map.get(key) ?? []).filter((t) => now - t < windowMs); + if (fresh.length >= max) { + const retryAfterSec = Math.ceil((windowMs - (now - Math.min(...fresh))) / 1000); + map.set(key, fresh); return { ok: false, retryAfterSec: Math.max(1, retryAfterSec) }; } fresh.push(now); - hits.set(userId, fresh); + map.set(key, fresh); return { ok: true }; } + +export function rateLimitDomainLookup(userId: string) { + return consume(lookupHits, userId, MAX_REQUESTS, WINDOW_MS); +} + +/** Limit explicit checks per user and per domain to keep outbound lookups bounded. */ +export function rateLimitDomainCheck(userId: string, domainId: string) { + const user = consume(checkHits, userId, MAX_REQUESTS, WINDOW_MS); + if (!user.ok) return user; + return consume(domainHits, `${userId}:${domainId}`, MAX_DOMAIN_REQUESTS, DOMAIN_WINDOW_MS); +} + +/** Process-local overlap guard; prevents duplicate concurrent work for one domain. */ +export function acquireDomainCheck(userId: string, domainId: string): (() => void) | null { + const key = `${userId}:${domainId}`; + if (inFlightChecks.has(key)) return null; + inFlightChecks.add(key); + return () => inFlightChecks.delete(key); +} diff --git a/src/repositories/domainCheckRun.repository.ts b/src/repositories/domainCheckRun.repository.ts new file mode 100644 index 0000000..183c723 --- /dev/null +++ b/src/repositories/domainCheckRun.repository.ts @@ -0,0 +1,15 @@ +import { and, desc, eq } from "drizzle-orm"; +import { db } from "../db/client"; +import { domainCheckRuns, domains } from "../db/schema"; +import type { DomainEnrichment } from "../domains/enrichment/types"; + +export type DomainCheckRun = typeof domainCheckRuns.$inferSelect; + +export async function findDomainCheckRunsForUser(domainId: string, userId: string, limit = 20) { + const [owned] = await db.select({ id: domains.id }).from(domains) + .where(and(eq(domains.id, domainId), eq(domains.userId, userId))).limit(1); + if (!owned) return null; + return db.select().from(domainCheckRuns) + .where(eq(domainCheckRuns.domainId, domainId)) + .orderBy(desc(domainCheckRuns.checkedAt)).limit(limit); +} diff --git a/src/services/domainCheck.service.ts b/src/services/domainCheck.service.ts new file mode 100644 index 0000000..e6016a1 --- /dev/null +++ b/src/services/domainCheck.service.ts @@ -0,0 +1,98 @@ +import type { ApiResult } from "../types/api"; +import type { DomainEnrichment } from "../domains/enrichment/types"; +import { enrichDomain } from "../domains/enrichment/service"; +import { enrichmentToStorable } from "../models/domain.model"; +import { findDomainByIdForUser } from "../repositories/domain.repository"; +import { findDomainCheckRunsForUser } from "../repositories/domainCheckRun.repository"; +import { acquireDomainCheck, rateLimitDomainCheck } from "../domains/enrichment/ratelimit"; +import { and, eq } from "drizzle-orm"; +import { db } from "../db/client"; +import { domainCheckRuns, domains } from "../db/schema"; + +const HISTORY_LIMIT = 20; + +function unauthorized(): ApiResult { return { status: 401, body: { error: "Unauthorized" } }; } +function notFound(): ApiResult { return { status: 404, body: { error: "Not found" } }; } + + +export async function refreshDomainCheckForUser(userId: string | null, domainId: string): Promise { + if (!userId) return unauthorized(); + if (!domainId) return notFound(); + + let domain: Awaited>; + try { + domain = await findDomainByIdForUser(domainId, userId); + } catch (error) { + return { status: 500, body: { error: error instanceof Error ? error.message : "Database error" } }; + } + if (!domain) return notFound(); + + const releaseCheck = acquireDomainCheck(userId, domainId); + if (!releaseCheck) return { status: 409, body: { error: "A check for this domain is already running." } }; + const limited = rateLimitDomainCheck(userId, domainId); + if (!limited.ok) { + releaseCheck(); + return { status: 429, body: { + error: "This domain was checked recently. Try again shortly.", + retryAfter: limited.retryAfterSec, + } }; + } + + const checkedAt = new Date(); + let result: DomainEnrichment; + let errorMessage: string | null = null; + try { + result = enrichmentToStorable(await enrichDomain(domain.hostname)); + } catch { + // Record only sanitized failure information; raw dependency errors may contain URLs or details. + errorMessage = "Domain check failed"; + result = { fetchedAt: checkedAt.toISOString(), errors: [errorMessage] }; + } + + try { + // The append-only history row and latest snapshot update commit or roll back together. + const suggested = result.suggested; + const suggestedExpiry = suggested?.expiresAt ? new Date(suggested.expiresAt) : null; + const expiry = domain.expiresAt ?? (suggestedExpiry && !Number.isNaN(suggestedExpiry.getTime()) ? suggestedExpiry : null); + const [run] = await db.transaction(async (tx) => { + const [inserted] = await tx.insert(domainCheckRuns).values({ + id: crypto.randomUUID(), domainId, checkedAt, result, error: errorMessage, + }).returning(); + await tx.update(domains).set({ + enrichment: result, + enrichedAt: checkedAt, + expiresAt: expiry, + registrarName: domain.registrarName ?? suggested?.registrarName ?? null, + updatedAt: checkedAt, + }).where(and(eq(domains.id, domainId), eq(domains.userId, userId))); + return [inserted]; + }); + + return { status: 200, body: { run: { + id: run.id, checkedAt: run.checkedAt.toISOString(), result: run.result, error: run.error, + } } }; + } catch { + return { status: 500, body: { error: "Could not persist domain check snapshot" } }; + } finally { + releaseCheck(); + } +} + +export async function listDomainCheckRunsForUser( + userId: string | null, domainId: string, limitRaw?: string | null +): Promise { + if (!userId) return unauthorized(); + const limit = limitRaw ? Number(limitRaw) : HISTORY_LIMIT; + if (!Number.isInteger(limit) || limit < 1 || limit > 100) { + return { status: 400, body: { error: "limit must be an integer between 1 and 100" } }; + } + try { + const runs = await findDomainCheckRunsForUser(domainId, userId, limit); + if (!runs) return notFound(); + return { status: 200, body: { runs: runs.map((run) => ({ + id: run.id, checkedAt: run.checkedAt.toISOString(), result: run.result, error: run.error, + })) } }; + } catch (error) { + return { status: 500, body: { error: error instanceof Error ? error.message : "Database error" } }; + } +}