diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2374ea..92ec529 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,8 +50,23 @@ jobs: # that the host-arch test build would miss. run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ./cmd/cosift + - name: Start Firestore emulator + # Pinned by digest (emulators tag); the emulator tests skip without it. + run: | + docker run -d --name firestore-emulator -p 8080:8080 \ + gcr.io/google.com/cloudsdktool/google-cloud-cli@sha256:45a15cc163d2df13137478856b4b9bb90426100fa8aec542269460c10110b727 \ + gcloud emulators firestore start --host-port=0.0.0.0:8080 + for i in $(seq 1 60); do + curl -fsS http://localhost:8080 >/dev/null 2>&1 && exit 0 + sleep 2 + done + docker logs firestore-emulator + exit 1 + - name: Run tests with coverage # cosift's internal/crawler hits ~25s; allow 5m total headroom. + env: + FIRESTORE_EMULATOR_HOST: localhost:8080 run: go test -race -coverprofile=coverage.out -covermode=atomic -timeout 10m ./... - name: Upload coverage to Codecov diff --git a/cmd/cosift/backfill_content_sha_rerun_test.go b/cmd/cosift/backfill_content_sha_rerun_test.go new file mode 100644 index 0000000..0edfa77 --- /dev/null +++ b/cmd/cosift/backfill_content_sha_rerun_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "crypto/sha256" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pilot-protocol/cosift/internal/store" +) + +func captureBackfillStderr(t *testing.T, f func()) string { + t.Helper() + orig := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = w + defer func() { os.Stderr = orig }() + f() + w.Close() + out, _ := io.ReadAll(r) + return string(out) +} + +func seedBackfillStore(t *testing.T, texts ...string) string { + t.Helper() + dir := t.TempDir() + ps, err := store.OpenPebble(dir) + if err != nil { + t.Fatal(err) + } + for _, text := range texts { + sum := sha256.Sum256([]byte(text)) + if _, err := ps.UpsertDocument(context.Background(), &store.Document{URL: "https://example.com/" + text, Title: text, Text: text, ContentSHA: sum[:]}); err != nil { + t.Fatal(err) + } + } + if err := ps.DB().DeleteRange([]byte{'C'}, []byte{'D'}, nil); err != nil { + t.Fatal(err) + } + ps.Close() + return dir +} + +func TestBackfillContentSHALeavesACompleteIndexAloneUnlessRescanned(t *testing.T) { + dir := seedBackfillStore(t, "one", "two") + ctx := context.Background() + first := captureBackfillStderr(t, func() { + if err := runBackfillContentSHA(ctx, []string{"-dir", dir}); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(first, "scanned 2 documents, indexed 2") { + t.Fatalf("first pass output %q", first) + } + second := captureBackfillStderr(t, func() { + if err := runBackfillContentSHA(ctx, []string{"-dir", dir}); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(second, "already complete") || !strings.Contains(second, "-rescan") || strings.Contains(second, "scanned") { + t.Fatalf("second pass output %q", second) + } + rescan := captureBackfillStderr(t, func() { + if err := runBackfillContentSHA(ctx, []string{"-dir", dir, "-rescan"}); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(rescan, "scanned 2 documents, indexed 2") { + t.Fatalf("rescan output %q", rescan) + } +} + +func TestBackfillContentSHAReportsHowToResumeAfterAnInterruption(t *testing.T) { + dir := seedBackfillStore(t, "one") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := runBackfillContentSHA(ctx, []string{"-dir", dir}) + if err == nil || !strings.Contains(err.Error(), "rerun to resume") || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("interrupted backfill: %v", err) + } + ps, err := store.OpenPebble(dir) + if err != nil { + t.Fatal(err) + } + defer ps.Close() + if done, _ := ps.ContentIndexComplete(context.Background()); done { + t.Fatal("an interrupted command marked the index complete") + } +} + +func TestBackfillContentSHAFailsWhenTheStoreCannotBeOpened(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + err := runBackfillContentSHA(context.Background(), []string{"-dir", filepath.Join(blocker, "pebble")}) + if err == nil || !strings.Contains(err.Error(), "open store") { + t.Fatalf("unopenable store: %v", err) + } +} diff --git a/cmd/cosift/community_moderation_gates_test.go b/cmd/cosift/community_moderation_gates_test.go new file mode 100644 index 0000000..2633f67 --- /dev/null +++ b/cmd/cosift/community_moderation_gates_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/pilot-protocol/cosift/internal/community" +) + +func TestCommunityModerationDecidesObviousCasesWithoutTheModel(t *testing.T) { + long := strings.Repeat("A neutral paragraph about reliable software systems. ", 5) + for name, tc := range map[string]struct { + doc community.ModerationDocument + want community.ModerationVerdict + }{ + "adult host": {community.ModerationDocument{URL: "https://www.pornhub.com/view", Title: "Some Video", Text: long}, + community.ModerationVerdict{Decision: "reject", Category: "adult"}}, + "adult content on a neutral host": {community.ModerationDocument{URL: "https://generic.example.net/g", Title: "Gallery", Text: "creampie and gangbang scenes"}, + community.ModerationVerdict{Decision: "reject", Category: "adult"}}, + "bot-check page": {community.ModerationDocument{URL: "https://example.com/check", Title: "Just a moment", Text: long}, + community.ModerationVerdict{Decision: "uncertain", Category: "unverified"}}, + "parked domain": {community.ModerationDocument{URL: "https://example.com/parked", Title: "Domain for sale", Text: long}, + community.ModerationVerdict{Decision: "reject", Category: "low_quality"}}, + } { + t.Run(name, func(t *testing.T) { + s := &pebbleHTTP{} + reserved := false + got, err := s.moderateDocument(context.Background(), tc.doc, func() error { reserved = true; return nil }) + if err != nil || got != tc.want { + t.Fatalf("verdict %+v, %v; want %+v", got, err, tc.want) + } + if reserved { + t.Fatal("a document decided without the model still took from its budget") + } + }) + } +} diff --git a/cmd/cosift/community_receipt_errors_test.go b/cmd/cosift/community_receipt_errors_test.go new file mode 100644 index 0000000..64471bf --- /dev/null +++ b/cmd/cosift/community_receipt_errors_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "strings" + "testing" + + "github.com/pilot-protocol/cosift/internal/crawler" + "github.com/pilot-protocol/cosift/internal/store" +) + +func TestCommunityReceiptKeepsTheJournalWhenTheContextEndsAfterIndexing(t *testing.T) { + db := openReceiptStore(t) + completeContentIndex(t, db) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + s := newReceiptServer(db, func(ctx context.Context, raw string, _ *crawler.LocalArtifact, _ string) (crawler.ContributionReceipt, error) { + calls++ + novel := calls == 1 + _ = crawler.ReportNovelty(ctx, crawler.NoveltyDecision{FinalURL: raw, Novel: novel}) + cancel() + return crawler.ContributionReceipt{Indexed: true, Novel: novel, ContentHash: strings.Repeat("a", 64)}, nil + }) + approved := strings.Repeat("a", 64) + if _, err := s.fetchCommunityReceipt(ctx, "cancelled-after-index-1", "https://example.com/late", nil, approved); !errors.Is(err, context.Canceled) { + t.Fatalf("expected the cancellation, got %v", err) + } + r, err := s.fetchCommunityReceipt(context.Background(), "cancelled-after-index-1", "https://example.com/late", nil, approved) + if err != nil || !r.Novel || !r.Indexed { + t.Fatalf("retry lost the recorded verdict: %+v %v", r, err) + } + if calls != 2 { + t.Fatalf("fetch calls = %d, want 2 (the first attempt never journaled a receipt)", calls) + } +} + +func TestCommunityReceiptReplayFailsOnACancelledContext(t *testing.T) { + s := newReceiptServer(openReceiptStore(t), nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + record := communityReceiptRecord{Receipt: &crawler.ContributionReceipt{Indexed: true, Novel: true}} + if _, err := s.replayCommunityReceipt(ctx, record, "https://example.com/a"); !errors.Is(err, context.Canceled) { + t.Fatalf("replay on a cancelled context: %v", err) + } +} + +func TestCommunityReceiptReplayFailsWhenTheLegacyURLCannotBeJudged(t *testing.T) { + db := openReceiptStore(t) + completeContentIndex(t, db) + s := newReceiptServer(db, nil) + record := communityReceiptRecord{Receipt: &crawler.ContributionReceipt{Indexed: true, Novel: true, ContentHash: strings.Repeat("a", 64)}} + if r, err := s.replayCommunityReceipt(context.Background(), record, "://not a url"); err == nil || r.Novel { + t.Fatalf("an unjudgeable legacy receipt stayed novel: %+v %v", r, err) + } +} + +func TestContentUnclaimed(t *testing.T) { + db := openReceiptStore(t) + completeContentIndex(t, db) + ctx := context.Background() + s := newReceiptServer(db, nil) + text := "unclaimed body" + sum := sha256.Sum256([]byte(text)) + if _, err := db.UpsertDocument(ctx, &store.Document{URL: "https://example.com/mine", Title: "Guide", Text: text, ContentSHA: sum[:]}); err != nil { + t.Fatal(err) + } + receipt := func(final, hash string) crawler.ContributionReceipt { + return crawler.ContributionReceipt{FinalURL: final, ContentHash: hash} + } + good := hex.EncodeToString(sum[:]) + cancelled, cancel := context.WithCancel(ctx) + cancel() + + for name, tc := range map[string]struct { + ctx context.Context + raw string + receipt crawler.ContributionReceipt + want bool + wantErr bool + }{ + "sole holder": {ctx, "https://example.com/mine", receipt("", good), true, false}, + "the final URL wins over raw": {ctx, "https://example.com/other", receipt("https://example.com/mine", good), true, false}, + "unstored document": {ctx, "https://example.com/missing", receipt("", good), false, false}, + "content hash is not hex": {ctx, "https://example.com/mine", receipt("", "not-hex"), false, false}, + "raw URL that cannot be judged": {ctx, "://not a url", receipt("", good), false, true}, + "cancelled context": {cancelled, "https://example.com/mine", receipt("", good), false, true}, + } { + t.Run(name, func(t *testing.T) { + got, err := s.contentUnclaimed(tc.ctx, tc.raw, tc.receipt) + if got != tc.want || (err != nil) != tc.wantErr { + t.Fatalf("contentUnclaimed = %v, %v; want %v, err=%v", got, err, tc.want, tc.wantErr) + } + }) + } +} diff --git a/cmd/cosift/serve_v1_ceiling_failure_test.go b/cmd/cosift/serve_v1_ceiling_failure_test.go new file mode 100644 index 0000000..0e14601 --- /dev/null +++ b/cmd/cosift/serve_v1_ceiling_failure_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "errors" + "testing" + + "github.com/cockroachdb/pebble" +) + +func TestModerationReservationRefusesWithoutACounter(t *testing.T) { + s := &pebbleHTTP{} + err := s.reserveCommunityModeration() + var ceiling *ceilingError + if err == nil || errors.As(err, &ceiling) { + t.Fatalf("a missing store must fail as an error, not as the daily ceiling: %v", err) + } +} + +func TestModerationReservationFailsOnACorruptCounterWithoutSpendingTheAllowance(t *testing.T) { + s, _ := ceilingServer(t, 5, nil) + if err := s.store.DB().Set(communityModerationsKey, []byte("not json"), pebble.Sync); err != nil { + t.Fatal(err) + } + err := s.reserveCommunityModeration() + var ceiling *ceilingError + if err == nil || errors.As(err, &ceiling) { + t.Fatalf("corrupt counter: %v", err) + } + got, closer, gerr := s.store.DB().Get(communityModerationsKey) + if gerr != nil { + t.Fatal(gerr) + } + defer closer.Close() + if string(got) != "not json" { + t.Fatalf("the counter was overwritten: %q", got) + } +} diff --git a/internal/community/articles_test.go b/internal/community/articles_test.go index 3709807..f536d8f 100644 --- a/internal/community/articles_test.go +++ b/internal/community/articles_test.go @@ -60,3 +60,16 @@ func TestEngineArticlesClassifyTheEnginesAnswers(t *testing.T) { t.Fatal("a non-ULID id reached the engine") } } + +func TestEngineArticlesReportAnUnreachableEngineWithoutItsAddress(t *testing.T) { + engine := httptest.NewServer(http.NotFoundHandler()) + client, err := newEngineClient(engine.URL, wikiTestKey) + if err != nil { + t.Fatal(err) + } + engine.Close() + _, err = engineArticles{client}.ArticleByID(context.Background(), testRef) + if err == nil || errors.Is(err, ErrArticleGone) || strings.Contains(err.Error(), engine.URL) { + t.Fatalf("unreachable engine: %v", err) + } +} diff --git a/internal/community/awards_edge_test.go b/internal/community/awards_edge_test.go new file mode 100644 index 0000000..d6d4eef --- /dev/null +++ b/internal/community/awards_edge_test.go @@ -0,0 +1,221 @@ +package community + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/cosift/internal/sharedaccount" +) + +func TestValidateAwardNamesTheFirstBrokenRule(t *testing.T) { + r := newAwardRig(t) + article := r.articleAward(1, 1, 1000) + royalty := r.royalty(2, 5, "2026-10-19") + clawback := r.clawback(3, 1, 50) + for name, tc := range map[string]struct { + base sharedaccount.Award + mutate func(*sharedaccount.Award) + want string + }{ + "article is valid": {article, func(*sharedaccount.Award) {}, ""}, + "royalty is valid": {royalty, func(*sharedaccount.Award) {}, ""}, + "clawback is valid": {clawback, func(*sharedaccount.Award) {}, ""}, + "unknown reason": {article, func(a *sharedaccount.Award) { a.Reason = "bonus" }, "reason"}, + "royalty day disagrees": {royalty, func(a *sharedaccount.Award) { a.Ref.Day = "2026-10-18" }, "id_form"}, + "royalty article disagrees": {royalty, func(a *sharedaccount.Award) { a.Ref.ArticleID = ulidN(5) }, "id_form"}, + "royalty contributor": {royalty, func(a *sharedaccount.Award) { a.CID = cidN(9) }, "id_form"}, + "royalty id malformed": {royalty, func(a *sharedaccount.Award) { a.ID = "roy:today" }, "id_form"}, + "royalty above the cap": {royalty, func(a *sharedaccount.Award) { a.Amount = 101 }, "amount"}, + "royalty at the cap": {royalty, func(a *sharedaccount.Award) { a.Amount = 100 }, ""}, + "clawback request disagrees": {clawback, func(a *sharedaccount.Award) { a.Ref.RequestID = ulidN(1) + "_" + ulidN(2) }, "id_form"}, + "clawback contributor": {clawback, func(a *sharedaccount.Award) { a.CID = cidN(9) }, "id_form"}, + "clawback article": {clawback, func(a *sharedaccount.Award) { a.Ref.ArticleID = "short" }, "id_form"}, + "clawback positive": {clawback, func(a *sharedaccount.Award) { a.Amount = 1 }, "amount"}, + "clawback below the floor": {clawback, func(a *sharedaccount.Award) { a.Amount = -(1 << 32) }, "amount"}, + "revision id on an article": {article, func(a *sharedaccount.Award) { a.ID = "rev:" + a.Ref.ContributionID }, "id_form"}, + "article ref is not a ulid": {article, func(a *sharedaccount.Award) { a.Ref.ArticleID = "short" }, "ref"}, + "award already applied": {article, func(a *sharedaccount.Award) { a.Status = "applied" }, "state"}, + } { + t.Run(name, func(t *testing.T) { + a := tc.base + tc.mutate(&a) + got := validateAward(a) + if got != tc.want { + t.Fatalf("validateAward = %q, want %q", got, tc.want) + } + }) + } +} + +func TestLedgerReasonNamesTheAwardKind(t *testing.T) { + for reason, want := range map[string]string{"article": "contribution_award", "revision": "contribution_award", "royalty": "royalty", "clawback": "clawback"} { + if got := ledgerReason(reason); got != want { + t.Errorf("ledgerReason(%q) = %q, want %q", reason, got, want) + } + } +} + +func TestPollSurfacesASourceFailureAndTheNextPollRecovers(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.src.pendingErr = sharedaccount.ErrUnavailable + if err := r.s.pollAwards(context.Background()); !errors.Is(err, sharedaccount.ErrUnavailable) { + t.Fatalf("poll error = %v", err) + } + if r.ledgerRows(t) != 0 { + t.Fatal("an award was applied from a failed read") + } + r.src.pendingErr = nil + r.tick(t) + if _, found := r.ledger(t, a.ID); !found { + t.Fatal("the poll did not recover") + } +} + +func TestPollStopsAtACancelledContextWithoutApplyingAnything(t *testing.T) { + r := newAwardRig(t) + r.articleAward(1, 1, 1000) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := r.s.pollAwards(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("poll error = %v", err) + } + if r.ledgerRows(t) != 0 || r.src.calls["MarkApplied"] != 0 { + t.Fatalf("a cancelled poll applied %d rows, %d marks", r.ledgerRows(t), r.src.calls["MarkApplied"]) + } +} + +func TestRetryStateExpiresAfterTwoHoursAndStaysBounded(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.src.contribErr = sharedaccount.ErrUnavailable + r.tick(t) + tracked := func() int { + r.s.awards.mu.Lock() + defer r.s.awards.mu.Unlock() + return len(r.s.awards.tries) + } + if tracked() != 1 { + t.Fatalf("tracked = %d after a failure", tracked()) + } + delete(r.src.awards, a.ID) + r.now = r.now.Add(time.Hour) + r.tick(t) + if tracked() != 1 { + t.Fatal("back-off state dropped within two hours") + } + r.now = r.now.Add(time.Hour + time.Minute) + r.tick(t) + if tracked() != 0 { + t.Fatal("back-off state for a vanished award was kept") + } + + p := newAwardPoller(r.src, r.art) + for i := 0; i < awardMemory+25; i++ { + p.fail(fmt.Sprintf("award-%d", i)) + } + if len(p.tries) != awardMemory { + t.Fatalf("retry state holds %d entries, want the cap %d", len(p.tries), awardMemory) + } +} + +func TestAwardStaysPendingWhenTheLedgerCannotBeRead(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + delete(r.art.rec, a.Ref.ArticleID) + r.s.db.Close() + r.tick(t) + if r.src.result(a.ID) != nil || r.src.calls["MarkApplied"] != 0 { + t.Fatalf("a skip was written without knowing whether the award was credited: %+v", r.src.result(a.ID)) + } + if !strings.Contains(r.logs.String(), "award_unresolved") || !strings.Contains(r.logs.String(), "class=ledger") { + t.Fatalf("log %q", r.logs.String()) + } +} + +func TestAwardStaysPendingWhenTheAccountCannotBeLinked(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.s.db.Close() + r.tick(t) + if r.src.result(a.ID) != nil || !strings.Contains(r.logs.String(), "class=link") { + t.Fatalf("result %+v log %q", r.src.result(a.ID), r.logs.String()) + } +} + +func TestAwardStaysPendingWhenTheLedgerWriteFails(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.s.awards.hook = func(step string) error { + if step == "insert" { + _, err := r.s.db.Exec(`DROP TABLE credit_ledger`) + return err + } + return nil + } + r.tick(t) + if r.src.result(a.ID) != nil || r.src.calls["MarkApplied"] != 0 || !strings.Contains(r.logs.String(), "class=ledger") { + t.Fatalf("result %+v log %q", r.src.result(a.ID), r.logs.String()) + } +} + +func TestAwardIsNeverMarkedAppliedWhenItsLedgerRowIsMissing(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.s.awards.hook = func(step string) error { + if step == "insert" { + _, err := r.s.db.Exec(`CREATE TRIGGER drop_award BEFORE INSERT ON credit_ledger BEGIN SELECT RAISE(IGNORE); END`) + return err + } + return nil + } + r.tick(t) + if r.src.result(a.ID) != nil || r.src.calls["MarkApplied"] != 0 || !strings.Contains(r.logs.String(), "class=ledger") { + t.Fatalf("result %+v log %q", r.src.result(a.ID), r.logs.String()) + } +} + +func TestWeekOldAwardIsNotExpiredWhenTheLedgerCannotBeRead(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + r.src.accountErr[uidN(1)] = sharedaccount.ErrUnavailable + r.src.awards[a.ID].a.CreatedAt = r.now.Add(-8 * 24 * time.Hour) + r.tick(t) + r.s.db.Close() + r.now = r.now.Add(2 * time.Hour) + r.tick(t) + if r.src.result(a.ID) != nil || r.src.calls["MarkApplied"] != 0 { + t.Fatalf("expired without checking the ledger: %+v", r.src.result(a.ID)) + } +} + +func TestRunAwardsLogsAFailedPollWithoutItsDetail(t *testing.T) { + old := awardPollEvery + awardPollEvery = 10 * time.Millisecond + t.Cleanup(func() { awardPollEvery = old }) + r := newAwardRig(t) + r.src.pendingErr = errors.New("rpc detail with a credential-like value") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); r.s.runAwards(ctx) }() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + r.src.mu.Lock() + n := r.src.calls["PendingAwards"] + r.src.mu.Unlock() + if n >= 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + out := r.logs.String() + if !strings.Contains(out, "community: award poll failed") || strings.Contains(out, "credential-like") { + t.Fatalf("log %q", out) + } +} diff --git a/internal/community/credit_guidance_test.go b/internal/community/credit_guidance_test.go new file mode 100644 index 0000000..86b6dda --- /dev/null +++ b/internal/community/credit_guidance_test.go @@ -0,0 +1,55 @@ +package community + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestInsufficientCreditsGuidanceListsOnlyTheWaysThatExist(t *testing.T) { + for name, tc := range map[string]struct { + payments, reward bool + want, absent []string + }{ + "monthly grant only": {false, false, []string{"use your next monthly grant"}, []string{"Billing", "contribute"}}, + "with billing": {true, false, []string{"add credits in Billing", "use your next monthly grant"}, []string{"contribute"}}, + "with rewards": {false, true, []string{"contribute verified new webpages"}, []string{"Billing"}}, + } { + t.Run(name, func(t *testing.T) { + s := testServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{}`)) })) + if tc.payments { + s.cfg.StripeSecretKey, s.cfg.StripeWebhookSecret, s.cfg.AllowTestPayments = "sk_test_fake_for_unit_tests", "whsec_fake_for_unit_tests", true + } + s.cfg.ContributionReward = tc.reward + _, cookie := fundedCreditAccount(t, s, 0) + w := request(t, s, "GET", "/api/search?q=test", nil, cookie) + expect(t, w, 429) + for _, want := range tc.want { + if !strings.Contains(w.Body.String(), want) { + t.Errorf("guidance lacks %q: %s", want, w.Body.String()) + } + } + for _, absent := range tc.absent { + if strings.Contains(w.Body.String(), absent) { + t.Errorf("guidance offers %q although it is unavailable: %s", absent, w.Body.String()) + } + } + }) + } +} + +func TestBearerLogoutClearsTheSessionCookieName(t *testing.T) { + s := httpsServer(t) + s.cfg.Shared = &fakeShared{} + req := httptest.NewRequest("POST", "/api/logout", strings.NewReader(`{}`)) + req.Header.Set("Authorization", "Bearer "+sharedTestToken) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Cosift-Client", "community") + w := httptest.NewRecorder() + s.ServeHTTP(w, req) + expect(t, w, 200) + if c := cookieNamed(w, s.sessionCookieName()); c == nil || c.MaxAge != -1 || c.Value != "" { + t.Fatalf("session cookie not cleared: %+v", c) + } +} diff --git a/internal/community/dispatch_failure_test.go b/internal/community/dispatch_failure_test.go new file mode 100644 index 0000000..1a6abfe --- /dev/null +++ b/internal/community/dispatch_failure_test.go @@ -0,0 +1,132 @@ +package community + +import ( + "context" + "testing" +) + +func refuseSubmissionUpdates(t *testing.T, s *Server) { + t.Helper() + if _, err := s.db.Exec(`CREATE TRIGGER refuse_status BEFORE UPDATE ON submissions BEGIN SELECT RAISE(ABORT, 'store offline'); END`); err != nil { + t.Fatal(err) + } +} + +func TestDispatchReportsAFailedStatusWriteAndLeavesTheSubmissionRetryable(t *testing.T) { + submit := func(t *testing.T, s *Server) { + t.Helper() + cookie := account(t, s, "member@example.com") + expect(t, request(t, s, "POST", "/api/submissions", map[string]any{"urls": []string{"https://example.com/guide"}}, cookie), 202) + } + for name, setup := range map[string]func(*testing.T) *Server{ + "backend unreachable": func(t *testing.T) *Server { + captureLog(t) + s := testServer(t, nil) + s.engine.base = refusedBackend(t) + s.backendDown.Store(true) + return s + }, + "broker key missing": func(t *testing.T) *Server { + captureLog(t) + s := testServer(t, nil) + s.engine.key = "" + return s + }, + "page check inconclusive": func(t *testing.T) *Server { + captureLog(t) + code := 500 + return routeServer(t, map[string]*int{"/v1/community/moderate": &code}) + }, + "delivery acknowledged": func(t *testing.T) *Server { + return routeServer(t, nil) + }, + } { + t.Run(name, func(t *testing.T) { + s := setup(t) + submit(t, s) + refuseSubmissionUpdates(t, s) + if err := s.dispatch(context.Background()); err == nil { + t.Fatal("a failed status write was swallowed") + } + if status, attempts := attemptsAndStatus(t, s); status != "pending" || attempts != 0 { + t.Fatalf("row left %s after %d attempts", status, attempts) + } + }) + } +} + +func TestDispatchReportsAFailedRequeueOfACrawlDelayedPage(t *testing.T) { + s := testServer(t, (&approvalBackend{}).handler(t)) + delayedSite(t, s, "5") + twoPages(t, s) + if err := s.dispatch(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`UPDATE submissions SET next_attempt=0 WHERE status='pending'`); err != nil { + t.Fatal(err) + } + refuseSubmissionUpdates(t, s) + if err := s.dispatch(context.Background()); err == nil { + t.Fatal("a failed requeue was swallowed") + } + var attempts int + s.db.QueryRow(`SELECT attempts FROM submissions WHERE status='pending'`).Scan(&attempts) + if attempts != 0 { + t.Fatalf("a held-back page used %d attempts", attempts) + } +} + +func TestDispatchKeepsASubmissionPendingWhenItsRewardCannotBeWritten(t *testing.T) { + s := rewardServer(t, func(c *Config) { c.ContributionReward = true }, novelReceipt(true)) + cookie := account(t, s, "member@example.com") + expect(t, request(t, s, "POST", "/api/submissions", map[string]any{"urls": []string{"https://docs.example.com/guide"}}, cookie), 202) + if _, err := s.db.Exec(`CREATE TRIGGER refuse_reward BEFORE INSERT ON credit_ledger WHEN NEW.reason='verified_contribution' BEGIN SELECT RAISE(ABORT, 'store offline'); END`); err != nil { + t.Fatal(err) + } + if err := s.dispatch(context.Background()); err == nil { + t.Fatal("a failed reward write was swallowed") + } + if status, _ := attemptsAndStatus(t, s); status != "pending" { + t.Fatalf("submission is %s although its reward was never recorded", status) + } + if _, err := s.db.Exec(`DROP TRIGGER refuse_reward`); err != nil { + t.Fatal(err) + } + s.db.Exec(`UPDATE submissions SET next_attempt=0`) + if err := s.dispatch(context.Background()); err != nil { + t.Fatal(err) + } + var userID string + s.db.QueryRow(`SELECT id FROM users WHERE email='member@example.com'`).Scan(&userID) + if status, _ := attemptsAndStatus(t, s); status != "indexed" || rewards(t, s, userID) != 1 { + t.Fatalf("retry: status %s, rewards %d", status, rewards(t, s, userID)) + } +} + +func TestOpenRefusesANonLoopbackEngine(t *testing.T) { + s, err := Open(Config{DataDir: t.TempDir(), Backend: "http://127.0.0.1:1", PublicURL: "http://localhost:7780", BrokerKey: testBrokerKey, EngineURL: "http://example.com:7779"}) + if err == nil { + s.Close() + t.Fatal("opened with an engine that is not on loopback") + } +} + +func TestLogoutReportsAFailedSessionDeleteAndKeepsTheSession(t *testing.T) { + s := testServer(t, nil) + cookie := account(t, s, "member@example.com") + if _, err := s.db.Exec(`CREATE TRIGGER refuse_logout BEFORE DELETE ON sessions BEGIN SELECT RAISE(ABORT, 'store offline'); END`); err != nil { + t.Fatal(err) + } + w := request(t, s, "POST", "/api/logout", map[string]string{}, cookie) + expect(t, w, 500) + for _, c := range w.Result().Cookies() { + if c.Name == cookie.Name && c.MaxAge < 0 { + t.Fatal("the cookie was cleared although the session survived") + } + } + var n int + s.db.QueryRow(`SELECT count(*) FROM sessions`).Scan(&n) + if n != 1 { + t.Fatalf("%d sessions", n) + } +} diff --git a/internal/community/engine_edge_test.go b/internal/community/engine_edge_test.go new file mode 100644 index 0000000..43e17e7 --- /dev/null +++ b/internal/community/engine_edge_test.go @@ -0,0 +1,61 @@ +package community + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +func TestEngineWithoutAKeySendsNothingAndCountsAsReachable(t *testing.T) { + c, err := newEngineClient("http://127.0.0.1:1", "") + if err != nil { + t.Fatal(err) + } + if _, err := c.post(context.Background(), enqueueRoute, []byte(`{}`), time.Second); !errors.Is(err, errNoBrokerKey) { + t.Fatalf("post without a key: %v", err) + } + if !c.reachable(context.Background()) { + t.Fatal("a missing key was reported as an unreachable engine") + } +} + +func TestHostGapsForgetHostsOlderThanTheLongestCrawlDelay(t *testing.T) { + var g hostGaps + now := time.Unix(1000, 0) + if wait := g.reserve("old.example", time.Second, now); wait != 0 { + t.Fatalf("first check waits %v", wait) + } + if wait := g.reserve("new.example", time.Second, now.Add(maxCrawlDelay)); wait != 0 { + t.Fatalf("second host waits %v", wait) + } + if _, kept := g.last["old.example"]; kept { + t.Fatal("a host older than the longest crawl delay was kept") + } + if wait := g.reserve("new.example", 5*time.Second, now.Add(maxCrawlDelay+time.Second)); wait != 4*time.Second { + t.Fatalf("a recent host must still be spaced: waits %v", wait) + } +} + +func TestRecordApprovalIgnoresAnonymousChecksAndLogsAFailedWrite(t *testing.T) { + s := testServer(t, nil) + logs := captureLog(t) + ctx := context.Background() + s.recordApproval(ctx, "", "hash") + var n int + s.db.QueryRow(`SELECT count(*) FROM submission_approvals`).Scan(&n) + if n != 0 || logs.String() != "" { + t.Fatalf("an approval without a submission was attempted: %d %q", n, logs.String()) + } + if _, err := s.db.Exec(`INSERT INTO submissions(id,url,created_at) VALUES('sub-1','https://example.com/a',0)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER refuse_approval BEFORE INSERT ON submission_approvals BEGIN SELECT RAISE(ABORT, 'store offline'); END`); err != nil { + t.Fatal(err) + } + s.recordApproval(ctx, "sub-1", "hash") + if !strings.Contains(logs.String(), "could not record a content approval") || s.approvalRecorded(ctx, "sub-1", "hash") { + t.Fatalf("log %q", logs.String()) + } +} diff --git a/internal/community/limiter_edge_test.go b/internal/community/limiter_edge_test.go new file mode 100644 index 0000000..0f86322 --- /dev/null +++ b/internal/community/limiter_edge_test.go @@ -0,0 +1,35 @@ +package community + +import ( + "testing" + "time" +) + +func TestLimiterRefusesEverythingAtAZeroLimitAndReportsTheWindow(t *testing.T) { + l := newLimiter(10) + now := time.Unix(1000, 0) + ok, until := l.take("k", 0, time.Minute, now) + if ok || !until.Equal(now.Add(time.Minute)) { + t.Fatalf("take = %v until %v", ok, until) + } + if len(l.keys) != 0 { + t.Fatal("a refused zero-limit call was remembered") + } +} + +func TestLimiterStartsAFreshWindowForAnExpiredKeyBehindALongerOne(t *testing.T) { + l := newLimiter(10) + now := time.Unix(1000, 0) + l.allow("long", 1, time.Hour, now) + l.allow("short", 1, time.Second, now) + later := now.Add(5 * time.Second) + if !l.allow("short", 1, time.Second, later) { + t.Fatal("an expired key stayed blocked behind a longer window") + } + if l.allow("short", 1, time.Second, later) { + t.Fatal("the fresh window allowed a second hit at limit 1") + } + if l.allow("long", 1, time.Hour, later) { + t.Fatal("the unexpired key was forgotten") + } +} diff --git a/internal/community/meter_failure_test.go b/internal/community/meter_failure_test.go new file mode 100644 index 0000000..3b949a5 --- /dev/null +++ b/internal/community/meter_failure_test.go @@ -0,0 +1,48 @@ +package community + +import ( + "strings" + "testing" +) + +func TestMeterAnswers500WithoutChargingWhenTheLedgerRefusesTheDebit(t *testing.T) { + s, uid := meterServer(t, 10) + logs := captureLog(t) + if _, err := s.db.Exec(`CREATE TRIGGER refuse_debit BEFORE INSERT ON credit_ledger WHEN NEW.reason='article_read' BEGIN SELECT RAISE(ABORT, 'ledger offline'); END`); err != nil { + t.Fatal(err) + } + w := meterCall(s, sharedTestToken, meterBody("refused-request-01"), nil) + expect(t, w, 500) + if a := decodeMeter(t, w); a.Code != "credits_unavailable" || a.Charged != 0 { + t.Fatalf("%+v", a) + } + if strings.Contains(w.Body.String(), "ledger offline") || strings.Contains(logs.String(), "ledger offline") { + t.Fatalf("the storage error leaked: body %q log %q", w.Body.String(), logs.String()) + } + if !strings.Contains(logs.String(), "credits_unavailable") || meterRows(t, s) != 0 || balanceOf(t, s, uid) != 10 { + t.Fatalf("rows %d balance %d log %q", meterRows(t, s), balanceOf(t, s, uid), logs.String()) + } +} + +func TestMeterAnswers500WhenTheMonthlyGrantCannotBeWritten(t *testing.T) { + s := testServer(t, nil) + s.cfg.Shared = &fakeShared{} + expect(t, bearerRequest(s, "/api/me", sharedTestToken), 200) + if _, err := s.db.Exec(`CREATE TRIGGER refuse_grant BEFORE INSERT ON credit_ledger WHEN NEW.reason='monthly_free' BEGIN SELECT RAISE(ABORT, 'ledger offline'); END`); err != nil { + t.Fatal(err) + } + w := meterCall(s, sharedTestToken, meterBody("ungranted-request1"), nil) + expect(t, w, 500) + if a := decodeMeter(t, w); a.Code != "credits_unavailable" || meterRows(t, s) != 0 { + t.Fatalf("%+v rows %d", a, meterRows(t, s)) + } +} + +func balanceOf(t *testing.T, s *Server, userID string) int { + t.Helper() + var b int + if err := s.db.QueryRow(`SELECT COALESCE(sum(delta),0) FROM credit_ledger WHERE user_id=?`, userID).Scan(&b); err != nil { + t.Fatal(err) + } + return b +} diff --git a/internal/community/shared_link_failure_test.go b/internal/community/shared_link_failure_test.go new file mode 100644 index 0000000..3c8222d --- /dev/null +++ b/internal/community/shared_link_failure_test.go @@ -0,0 +1,42 @@ +package community + +import ( + "strings" + "testing" + "time" +) + +func TestLinkingAnExistingPasswordAccountIsAllOrNothing(t *testing.T) { + for name, trigger := range map[string]string{ + "identity row": `CREATE TRIGGER refuse_link BEFORE INSERT ON shared_identities BEGIN SELECT RAISE(ABORT, 'store offline'); END`, + "session revoke": `CREATE TRIGGER refuse_link BEFORE DELETE ON sessions BEGIN SELECT RAISE(ABORT, 'store offline'); END`, + "password revoke": `CREATE TRIGGER refuse_link BEFORE UPDATE OF password_hash ON users BEGIN SELECT RAISE(ABORT, 'store offline'); END`, + } { + t.Run(name, func(t *testing.T) { + r := newAwardRig(t) + a := r.articleAward(1, 1, 1000) + account(t, r.s, emailN(1)) + if _, err := r.s.db.Exec(trigger); err != nil { + t.Fatal(err) + } + r.tick(t) + var links, sessions int + var hash string + r.s.db.QueryRow(`SELECT count(*) FROM shared_identities`).Scan(&links) + r.s.db.QueryRow(`SELECT count(*) FROM sessions`).Scan(&sessions) + r.s.db.QueryRow(`SELECT password_hash FROM users WHERE email=?`, emailN(1)).Scan(&hash) + if links != 0 || sessions != 1 || hash == "" { + t.Fatalf("a failed link was partly applied: links %d sessions %d password kept %v", links, sessions, hash != "") + } + if r.src.result(a.ID) != nil || r.ledgerRows(t) != 0 || !strings.Contains(r.logs.String(), "class=link") { + t.Fatalf("result %+v rows %d log %q", r.src.result(a.ID), r.ledgerRows(t), r.logs.String()) + } + r.s.db.Exec(`DROP TRIGGER refuse_link`) + r.now = r.now.Add(2 * time.Hour) + r.tick(t) + if _, found := r.ledger(t, a.ID); !found { + t.Fatal("the award was not applied once the store recovered") + } + }) + } +} diff --git a/internal/crawler/contribution_indexfail_test.go b/internal/crawler/contribution_indexfail_test.go new file mode 100644 index 0000000..037828a --- /dev/null +++ b/internal/crawler/contribution_indexfail_test.go @@ -0,0 +1,80 @@ +package crawler + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/pilot-protocol/cosift/internal/index" + "github.com/pilot-protocol/cosift/internal/store" +) + +type flakyContentIndex struct { + *store.PebbleStore + completeErr error + completeFrom int + calls int +} + +func (f *flakyContentIndex) ContentIndexComplete(ctx context.Context) (bool, error) { + f.calls++ + if f.completeErr != nil && f.calls >= f.completeFrom { + return false, f.completeErr + } + return f.PebbleStore.ContentIndexComplete(ctx) +} + +func flakyContributor(t *testing.T, routes map[string]string, flaky *flakyContentIndex) *Crawler { + t.Helper() + ps, err := store.OpenPebble(filepath.Join(t.TempDir(), "pebble")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ps.Close() }) + flaky.PebbleStore = ps + completeContentIndex(t, ps) + cfg := testCrawlerCfg() + cfg.RespectRobots = false + cfg.PerHostDelayMs = 0 + cfg.DisableLinkFollowing = true + c := NewWithBackend(cfg, flaky, index.NewPebbleBM25(ps)) + c.http.Transport = &siteRT{hits: map[string]int{}, routes: routes} + c.contributionOnce.Do(func() { c.contributionCrawler = c; c.contributionSlots = make(chan struct{}, 2) }) + return c +} + +func TestContributionFailsBeforeWritingWhenTheContentIndexCannotBeRead(t *testing.T) { + page := pagePlain("text the index cannot vouch for") + boom := errors.New("index unreadable") + flaky := &flakyContentIndex{completeErr: boom, completeFrom: 1} + c := flakyContributor(t, map[string]string{"https://news.example/a": page}, flaky) + r, err := c.FetchContribution(context.Background(), "https://news.example/a", nil, approvedForHTML(t, "https://news.example/a", page)) + if !errors.Is(err, boom) || r.Indexed || r.Novel { + t.Fatalf("receipt %+v err %v", r, err) + } + if doc, err := flaky.GetDocByURL(context.Background(), "https://news.example/a"); err == nil && doc != nil { + t.Fatal("document written although novelty could not be decided") + } +} + +func TestContributionReceiptFailsWhenTheIndexStateCannotBeReadAfterTheWrite(t *testing.T) { + page := pagePlain("text whose receipt cannot report the index state") + boom := errors.New("index unreadable") + flaky := &flakyContentIndex{completeErr: boom, completeFrom: 2} + c := flakyContributor(t, map[string]string{"https://news.example/b": page}, flaky) + r, err := c.FetchContribution(context.Background(), "https://news.example/b", nil, approvedForHTML(t, "https://news.example/b", page)) + if !errors.Is(err, boom) || r.Indexed || r.Novel || r.ContentIndexComplete { + t.Fatalf("a receipt without a known index state was returned: %+v %v", r, err) + } +} + +func TestRobotsCacheUsesTheDefaultBoundsForNonPositiveLimits(t *testing.T) { + r := NewRobotsWithLimits(nil, "ua", 0, -5) + if r.maxEntries != DefaultRobotsMaxEntries || r.maxBytes != DefaultRobotsMaxBytes { + t.Fatalf("bounds %d entries, %d bytes", r.maxEntries, r.maxBytes) + } + if r := NewRobotsWithLimits(nil, "ua", 7, 1234); r.maxEntries != 7 || r.maxBytes != 1234 { + t.Fatalf("explicit bounds not kept: %d, %d", r.maxEntries, r.maxBytes) + } +} diff --git a/internal/sharedaccount/awards_failures_test.go b/internal/sharedaccount/awards_failures_test.go new file mode 100644 index 0000000..312ddb1 --- /dev/null +++ b/internal/sharedaccount/awards_failures_test.go @@ -0,0 +1,54 @@ +package sharedaccount + +import ( + "context" + "errors" + "testing" + + pb "cloud.google.com/go/firestore/apiv1/firestorepb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type transactionFixture struct{ *firestoreFixture } + +func (transactionFixture) BeginTransaction(context.Context, *pb.BeginTransactionRequest) (*pb.BeginTransactionResponse, error) { + return &pb.BeginTransactionResponse{Transaction: []byte("fixture-transaction")}, nil +} + +func (transactionFixture) Rollback(context.Context, *pb.RollbackRequest) (*emptypb.Empty, error) { + return &emptypb.Empty{}, nil +} + +func deniedAwardSource(t *testing.T) AwardSource { + t.Helper() + store := googleFixtureStore(t, transactionFixture{&firestoreFixture{failure: true}}) + return NewAwardSource(store.db) +} + +func TestAwardReadsMapFirestoreFailuresToUnavailable(t *testing.T) { + src := deniedAwardSource(t) + ctx := context.Background() + if page, err := src.PendingAwards(ctx, AwardPosition{}, 10); !errors.Is(err, ErrUnavailable) || page != nil { + t.Fatalf("PendingAwards: %v %v", page, err) + } + if _, err := src.Contributor(ctx, "0f1e2d3c4b5a6978"); !errors.Is(err, ErrUnavailable) { + t.Fatalf("Contributor: %v", err) + } + if err := src.MarkApplied(ctx, "art:A", AwardResult{Status: "applied", AppliedDelta: 1000, LedgerID: "award:art:A"}); !errors.Is(err, ErrUnavailable) { + t.Fatalf("MarkApplied must not report success or a verdict it could not read: %v", err) + } +} + +func TestContributorWithoutAStringUIDIsUnavailable(t *testing.T) { + doc := &pb.Document{ + Name: "projects/fixture-project/databases/staging/documents/contributors/0f1e2d3c4b5a6978", + Fields: map[string]*pb.Value{"uid": {ValueType: &pb.Value_IntegerValue{IntegerValue: 7}}}, + CreateTime: timestamppb.Now(), + UpdateTime: timestamppb.Now(), + } + src := NewAwardSource(googleFixtureStore(t, transactionFixture{&firestoreFixture{account: doc}}).db) + if got, err := src.Contributor(context.Background(), "0f1e2d3c4b5a6978"); !errors.Is(err, ErrUnavailable) || got.UID != "" { + t.Fatalf("Contributor: %+v %v", got, err) + } +} diff --git a/internal/sharedaccount/google_failures_test.go b/internal/sharedaccount/google_failures_test.go index b23cb22..da1d350 100644 --- a/internal/sharedaccount/google_failures_test.go +++ b/internal/sharedaccount/google_failures_test.go @@ -18,7 +18,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func googleFixtureStore(t *testing.T, fixture *firestoreFixture) googleStore { +func googleFixtureStore(t *testing.T, fixture pb.FirestoreServer) googleStore { t.Helper() listener := bufconn.Listen(1 << 20) server := grpc.NewServer() diff --git a/internal/sharedaccount/google_version_test.go b/internal/sharedaccount/google_version_test.go new file mode 100644 index 0000000..c240837 --- /dev/null +++ b/internal/sharedaccount/google_version_test.go @@ -0,0 +1,34 @@ +package sharedaccount + +import ( + "errors" + "testing" +) + +func TestResolvedVersionAcceptsOnlyABoundedPositiveVersion(t *testing.T) { + const prefix = "projects/p/secrets/cosift-token-pepper/versions/" + for name, tc := range map[string]struct { + in string + want int + err error + }{ + "plain": {prefix + "7", 7, nil}, + "six digits": {prefix + "999999", 999999, nil}, + "project number": {"projects/123/secrets/cosift-token-pepper/versions/12", 12, nil}, + "alias not number": {prefix + "latest", 0, ErrUnavailable}, + "zero": {prefix + "0", 0, ErrUnavailable}, + "leading zero": {prefix + "07", 0, ErrUnavailable}, + "seven digits": {prefix + "1000000", 0, ErrUnavailable}, + "negative": {prefix + "-1", 0, ErrUnavailable}, + "no version": {"projects/p/secrets/cosift-token-pepper", 0, ErrUnavailable}, + "empty": {"", 0, ErrUnavailable}, + "trailing path": {prefix + "7/extra", 0, ErrUnavailable}, + } { + t.Run(name, func(t *testing.T) { + got, err := resolvedVersion(tc.in) + if got != tc.want || !errors.Is(err, tc.err) { + t.Fatalf("resolvedVersion(%q) = %d, %v; want %d, %v", tc.in, got, err, tc.want, tc.err) + } + }) + } +} diff --git a/internal/store/content_index_edge_test.go b/internal/store/content_index_edge_test.go new file mode 100644 index 0000000..9d9df9c --- /dev/null +++ b/internal/store/content_index_edge_test.go @@ -0,0 +1,200 @@ +package store + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/cockroachdb/pebble" +) + +type cancelAfterCalls struct { + context.Context + left int +} + +func (c *cancelAfterCalls) Err() error { + if c.left <= 0 { + return context.Canceled + } + c.left-- + return nil +} + +func cursorSaved(t *testing.T, p *PebbleStore) bool { + t.Helper() + _, closer, err := p.db.Get(metaKey(metaContentSHACursor)) + if err == nil { + closer.Close() + return true + } + if !errors.Is(err, pebble.ErrNotFound) { + t.Fatal(err) + } + return false +} + +func TestHasLiveContentIgnoresMalformedIndexKeys(t *testing.T) { + p := newPebbleStore(t) + sha := shaOf("indexed text") + if err := p.db.Set(append(contentSHAKey(sha, 0)[:1+len(sha)], 'x'), nil, pebble.Sync); err != nil { + t.Fatal(err) + } + mustLive(t, p, "indexed text", 0, false) + if _, err := p.UpsertDocument(context.Background(), contentDoc("https://example.com/a", "indexed text")); err != nil { + t.Fatal(err) + } + mustLive(t, p, "indexed text", 0, true) +} + +func TestHasLiveContentSurfacesACorruptDocument(t *testing.T) { + p := newPebbleStore(t) + sha := shaOf("vouched text") + if err := p.db.Set(contentSHAKey(sha, 99), nil, pebble.Sync); err != nil { + t.Fatal(err) + } + if err := p.db.Set(docKey(99), []byte("not a document"), pebble.Sync); err != nil { + t.Fatal(err) + } + live, err := p.HasLiveContent(context.Background(), sha, 0) + if err == nil || !strings.Contains(err.Error(), "decode doc 99") || live { + t.Fatalf("corrupt document: live=%v err=%v", live, err) + } +} + +func TestContentIndexCompleteHonoursCancellation(t *testing.T) { + p := newPebbleStore(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := p.ContentIndexComplete(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("ContentIndexComplete on a cancelled context: %v", err) + } + res, err := p.BackfillContentSHA(ctx, false, nil) + if !errors.Is(err, context.Canceled) || res.AlreadyComplete || res.Scanned != 0 { + t.Fatalf("backfill on a cancelled context: %+v %v", res, err) + } +} + +func TestBackfillContentSHACancelledBeforeAnyDocumentSavesNoCursor(t *testing.T) { + p := newPebbleStore(t) + ctx, cancel := context.WithCancel(context.Background()) + for _, text := range []string{"a", "b"} { + if _, err := p.UpsertDocument(ctx, contentDoc("https://example.com/"+text, text)); err != nil { + t.Fatal(err) + } + } + clearContentIndex(t, p) + cancel() + res, err := p.BackfillContentSHA(ctx, true, nil) + if !errors.Is(err, context.Canceled) || res.Scanned != 0 || cursorSaved(t, p) { + t.Fatalf("cancelled rescan: %+v %v cursor=%v", res, err, cursorSaved(t, p)) + } + if done, _ := p.ContentIndexComplete(context.Background()); done { + t.Fatal("cancelled rescan marked the index complete") + } +} + +func TestBackfillContentSHACommitsTheOpenBatchWhenCancelled(t *testing.T) { + p := newPebbleStore(t) + texts := []string{"a", "b", "c", "d", "e"} + for _, text := range texts { + if _, err := p.UpsertDocument(context.Background(), contentDoc("https://example.com/"+text, text)); err != nil { + t.Fatal(err) + } + } + clearContentIndex(t, p) + + first, err := p.BackfillContentSHA(&cancelAfterCalls{Context: context.Background(), left: 3}, false, nil) + if !errors.Is(err, context.Canceled) || first.Scanned != 2 || first.Indexed != 2 || !cursorSaved(t, p) { + t.Fatalf("interrupted pass: %+v %v cursor=%v", first, err, cursorSaved(t, p)) + } + if done, _ := p.ContentIndexComplete(context.Background()); done { + t.Fatal("interrupted pass marked the index complete") + } + mustLive(t, p, "b", 0, true) + mustLive(t, p, "c", 0, false) + + second, err := p.BackfillContentSHA(context.Background(), false, nil) + if err != nil || second.ResumedFrom != first.LastID || second.Scanned != 3 { + t.Fatalf("resume: %+v after %+v: %v", second, first, err) + } + for _, text := range texts { + mustLive(t, p, text, 0, true) + } +} + +func TestBackfillContentSHAIgnoresForeignKeysInTheDocumentFamily(t *testing.T) { + p := newPebbleStore(t) + ctx := context.Background() + id, err := p.UpsertDocument(ctx, contentDoc("https://example.com/a", "kept")) + if err != nil { + t.Fatal(err) + } + if err := p.db.Set(append(docKey(id+1), 'x'), []byte("not a document"), pebble.Sync); err != nil { + t.Fatal(err) + } + clearContentIndex(t, p) + res, err := p.BackfillContentSHA(ctx, false, nil) + if err != nil || res.Scanned != 1 || res.Indexed != 1 { + t.Fatalf("backfill %+v %v", res, err) + } + mustLive(t, p, "kept", 0, true) +} + +func TestBackfillContentSHAFailsOnACorruptDocumentWithoutCompleting(t *testing.T) { + p := newPebbleStore(t) + ctx := context.Background() + if _, err := p.UpsertDocument(ctx, contentDoc("https://example.com/a", "fine")); err != nil { + t.Fatal(err) + } + if err := p.db.Set(docKey(2), []byte("not a document"), pebble.Sync); err != nil { + t.Fatal(err) + } + clearContentIndex(t, p) + res, err := p.BackfillContentSHA(ctx, false, nil) + if err == nil || !strings.Contains(err.Error(), "decode doc 2") || res.Scanned != 1 { + t.Fatalf("corrupt document: %+v %v", res, err) + } + if done, _ := p.ContentIndexComplete(ctx); done { + t.Fatal("a failed pass marked the index complete") + } +} + +func TestBackfillContentSHAIgnoresAMalformedCursor(t *testing.T) { + p := newPebbleStore(t) + ctx := context.Background() + if _, err := p.UpsertDocument(ctx, contentDoc("https://example.com/a", "one")); err != nil { + t.Fatal(err) + } + clearContentIndex(t, p) + if err := p.db.Set(metaKey(metaContentSHACursor), []byte{1, 2, 3}, pebble.Sync); err != nil { + t.Fatal(err) + } + res, err := p.BackfillContentSHA(ctx, false, nil) + if err != nil || res.ResumedFrom != 0 || res.Scanned != 1 { + t.Fatalf("malformed cursor: %+v %v", res, err) + } +} + +func TestBackfillContentSHAReportsProgressAfterEachBatch(t *testing.T) { + old := contentSHABatch + contentSHABatch = 2 + t.Cleanup(func() { contentSHABatch = old }) + p := newPebbleStore(t) + ctx := context.Background() + for _, text := range []string{"a", "b", "c", "d", "e"} { + if _, err := p.UpsertDocument(ctx, contentDoc("https://example.com/"+text, text)); err != nil { + t.Fatal(err) + } + } + clearContentIndex(t, p) + var scanned []int64 + res, err := p.BackfillContentSHA(ctx, false, func(r ContentSHABackfill) { scanned = append(scanned, r.Scanned) }) + if err != nil || res.Scanned != 5 { + t.Fatalf("backfill %+v %v", res, err) + } + if len(scanned) != 3 || scanned[0] != 2 || scanned[1] != 4 || scanned[2] != 5 { + t.Fatalf("progress reports %v, want [2 4 5]", scanned) + } +} diff --git a/internal/store/content_variant_edge_test.go b/internal/store/content_variant_edge_test.go new file mode 100644 index 0000000..7613531 --- /dev/null +++ b/internal/store/content_variant_edge_test.go @@ -0,0 +1,59 @@ +package store + +import ( + "context" + "strings" + "testing" + + "github.com/cockroachdb/pebble" +) + +func TestHasUncomparableVariantRejectsAnUnparseableURL(t *testing.T) { + p := newPebbleStore(t) + if got, err := p.HasUncomparableVariant(context.Background(), "http://[::1", 0); err == nil || got { + t.Fatalf("unparseable URL: %v %v", got, err) + } +} + +func TestHasUncomparableVariantSurfacesAMalformedURLIndexValue(t *testing.T) { + p := newPebbleStore(t) + if err := p.db.Set(urlKey("https://news.example/guide"), []byte{1, 2, 3}, pebble.Sync); err != nil { + t.Fatal(err) + } + got, err := p.HasUncomparableVariant(context.Background(), "https://news.example/guide?ref=1", 0) + if err == nil || !strings.Contains(err.Error(), "malformed URL") || got { + t.Fatalf("malformed index value: %v %v", got, err) + } +} + +func TestHasUncomparableVariantSurfacesACorruptDocument(t *testing.T) { + p := newPebbleStore(t) + id, err := p.UpsertDocument(context.Background(), variantDoc("https://news.example/guide", "wet-import")) + if err != nil { + t.Fatal(err) + } + if err := p.db.Set(docKey(id), []byte("not a document"), pebble.Sync); err != nil { + t.Fatal(err) + } + got, err := p.HasUncomparableVariant(context.Background(), "https://news.example/guide?ref=1", 0) + if err == nil || !strings.Contains(err.Error(), "decode doc") || got { + t.Fatalf("corrupt document: %v %v", got, err) + } +} + +func TestHasUncomparableVariantSkipsMalformedPrefixEntries(t *testing.T) { + p := newPebbleStore(t) + ctx := context.Background() + if err := p.db.Set(urlKey("https://news.example/guide?broken"), []byte{1}, pebble.Sync); err != nil { + t.Fatal(err) + } + if got, err := p.HasUncomparableVariant(ctx, "https://news.example/guide", 0); err != nil || got { + t.Fatalf("malformed prefix entry: %v %v", got, err) + } + if _, err := p.UpsertDocument(ctx, variantDoc("https://news.example/guide?real", "wet-import")); err != nil { + t.Fatal(err) + } + if got, err := p.HasUncomparableVariant(ctx, "https://news.example/guide", 0); err != nil || !got { + t.Fatalf("a real entry behind a malformed one was missed: %v %v", got, err) + } +}