Tests for the v0.4.2 failure paths, and the Firestore emulator in CI

.github/workflows/ci.yml

@@ -50,8 +50,23 @@ jobs:
         # that the host-arch test build would miss.
         run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ./cmd/cosift
 
+      - name: Start Firestore emulator
+        # Pinned by digest (emulators tag); the emulator tests skip without it.
+        run: |
+          docker run -d --name firestore-emulator -p 8080:8080 \
+            gcr.io/google.com/cloudsdktool/google-cloud-cli@sha256:45a15cc163d2df13137478856b4b9bb90426100fa8aec542269460c10110b727 \
+            gcloud emulators firestore start --host-port=0.0.0.0:8080
+          for i in $(seq 1 60); do
+            curl -fsS http://localhost:8080 >/dev/null 2>&1 && exit 0
+            sleep 2
+          done
+          docker logs firestore-emulator
+          exit 1
+
       - name: Run tests with coverage
         # cosift's internal/crawler hits ~25s; allow 5m total headroom.
+        env:
+          FIRESTORE_EMULATOR_HOST: localhost:8080
         run: go test -race -coverprofile=coverage.out -covermode=atomic -timeout 10m ./...
 
       - name: Upload coverage to Codecov

cmd/cosift/backfill_content_sha_rerun_test.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func captureBackfillStderr(t *testing.T, f func()) string {
+	t.Helper()
+	orig := os.Stderr
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Stderr = w
+	defer func() { os.Stderr = orig }()
+	f()
+	w.Close()
+	out, _ := io.ReadAll(r)
+	return string(out)
+}
+
+func seedBackfillStore(t *testing.T, texts ...string) string {
+	t.Helper()
+	dir := t.TempDir()
+	ps, err := store.OpenPebble(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, text := range texts {
+		sum := sha256.Sum256([]byte(text))
+		if _, err := ps.UpsertDocument(context.Background(), &store.Document{URL: "https://example.com/" + text, Title: text, Text: text, ContentSHA: sum[:]}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := ps.DB().DeleteRange([]byte{'C'}, []byte{'D'}, nil); err != nil {
+		t.Fatal(err)
+	}
+	ps.Close()
+	return dir
+}
+
+func TestBackfillContentSHALeavesACompleteIndexAloneUnlessRescanned(t *testing.T) {
+	dir := seedBackfillStore(t, "one", "two")
+	ctx := context.Background()
+	first := captureBackfillStderr(t, func() {
+		if err := runBackfillContentSHA(ctx, []string{"-dir", dir}); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if !strings.Contains(first, "scanned 2 documents, indexed 2") {
+		t.Fatalf("first pass output %q", first)
+	}
+	second := captureBackfillStderr(t, func() {
+		if err := runBackfillContentSHA(ctx, []string{"-dir", dir}); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if !strings.Contains(second, "already complete") || !strings.Contains(second, "-rescan") || strings.Contains(second, "scanned") {
+		t.Fatalf("second pass output %q", second)
+	}
+	rescan := captureBackfillStderr(t, func() {
+		if err := runBackfillContentSHA(ctx, []string{"-dir", dir, "-rescan"}); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if !strings.Contains(rescan, "scanned 2 documents, indexed 2") {
+		t.Fatalf("rescan output %q", rescan)
+	}
+}
+
+func TestBackfillContentSHAReportsHowToResumeAfterAnInterruption(t *testing.T) {
+	dir := seedBackfillStore(t, "one")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	err := runBackfillContentSHA(ctx, []string{"-dir", dir})
+	if err == nil || !strings.Contains(err.Error(), "rerun to resume") || !strings.Contains(err.Error(), "context canceled") {
+		t.Fatalf("interrupted backfill: %v", err)
+	}
+	ps, err := store.OpenPebble(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ps.Close()
+	if done, _ := ps.ContentIndexComplete(context.Background()); done {
+		t.Fatal("an interrupted command marked the index complete")
+	}
+}
+
+func TestBackfillContentSHAFailsWhenTheStoreCannotBeOpened(t *testing.T) {
+	blocker := filepath.Join(t.TempDir(), "file")
+	if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	err := runBackfillContentSHA(context.Background(), []string{"-dir", filepath.Join(blocker, "pebble")})
+	if err == nil || !strings.Contains(err.Error(), "open store") {
+		t.Fatalf("unopenable store: %v", err)
+	}
+}

cmd/cosift/community_moderation_gates_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/pilot-protocol/cosift/internal/community"
+)
+
+func TestCommunityModerationDecidesObviousCasesWithoutTheModel(t *testing.T) {
+	long := strings.Repeat("A neutral paragraph about reliable software systems. ", 5)
+	for name, tc := range map[string]struct {
+		doc  community.ModerationDocument
+		want community.ModerationVerdict
+	}{
+		"adult host": {community.ModerationDocument{URL: "https://www.pornhub.com/view", Title: "Some Video", Text: long},
+			community.ModerationVerdict{Decision: "reject", Category: "adult"}},
+		"adult content on a neutral host": {community.ModerationDocument{URL: "https://generic.example.net/g", Title: "Gallery", Text: "creampie and gangbang scenes"},
+			community.ModerationVerdict{Decision: "reject", Category: "adult"}},
+		"bot-check page": {community.ModerationDocument{URL: "https://example.com/check", Title: "Just a moment", Text: long},
+			community.ModerationVerdict{Decision: "uncertain", Category: "unverified"}},
+		"parked domain": {community.ModerationDocument{URL: "https://example.com/parked", Title: "Domain for sale", Text: long},
+			community.ModerationVerdict{Decision: "reject", Category: "low_quality"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			s := &pebbleHTTP{}
+			reserved := false
+			got, err := s.moderateDocument(context.Background(), tc.doc, func() error { reserved = true; return nil })
+			if err != nil || got != tc.want {
+				t.Fatalf("verdict %+v, %v; want %+v", got, err, tc.want)
+			}
+			if reserved {
+				t.Fatal("a document decided without the model still took from its budget")
+			}
+		})
+	}
+}

cmd/cosift/community_receipt_errors_test.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/pilot-protocol/cosift/internal/crawler"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func TestCommunityReceiptKeepsTheJournalWhenTheContextEndsAfterIndexing(t *testing.T) {
+	db := openReceiptStore(t)
+	completeContentIndex(t, db)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	calls := 0
+	s := newReceiptServer(db, func(ctx context.Context, raw string, _ *crawler.LocalArtifact, _ string) (crawler.ContributionReceipt, error) {
+		calls++
+		novel := calls == 1
+		_ = crawler.ReportNovelty(ctx, crawler.NoveltyDecision{FinalURL: raw, Novel: novel})
+		cancel()
+		return crawler.ContributionReceipt{Indexed: true, Novel: novel, ContentHash: strings.Repeat("a", 64)}, nil
+	})
+	approved := strings.Repeat("a", 64)
+	if _, err := s.fetchCommunityReceipt(ctx, "cancelled-after-index-1", "https://example.com/late", nil, approved); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected the cancellation, got %v", err)
+	}
+	r, err := s.fetchCommunityReceipt(context.Background(), "cancelled-after-index-1", "https://example.com/late", nil, approved)
+	if err != nil || !r.Novel || !r.Indexed {
+		t.Fatalf("retry lost the recorded verdict: %+v %v", r, err)
+	}
+	if calls != 2 {
+		t.Fatalf("fetch calls = %d, want 2 (the first attempt never journaled a receipt)", calls)
+	}
+}
+
+func TestCommunityReceiptReplayFailsOnACancelledContext(t *testing.T) {
+	s := newReceiptServer(openReceiptStore(t), nil)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	record := communityReceiptRecord{Receipt: &crawler.ContributionReceipt{Indexed: true, Novel: true}}
+	if _, err := s.replayCommunityReceipt(ctx, record, "https://example.com/a"); !errors.Is(err, context.Canceled) {
+		t.Fatalf("replay on a cancelled context: %v", err)
+	}
+}
+
+func TestCommunityReceiptReplayFailsWhenTheLegacyURLCannotBeJudged(t *testing.T) {
+	db := openReceiptStore(t)
+	completeContentIndex(t, db)
+	s := newReceiptServer(db, nil)
+	record := communityReceiptRecord{Receipt: &crawler.ContributionReceipt{Indexed: true, Novel: true, ContentHash: strings.Repeat("a", 64)}}
+	if r, err := s.replayCommunityReceipt(context.Background(), record, "://not a url"); err == nil || r.Novel {
+		t.Fatalf("an unjudgeable legacy receipt stayed novel: %+v %v", r, err)
+	}
+}
+
+func TestContentUnclaimed(t *testing.T) {
+	db := openReceiptStore(t)
+	completeContentIndex(t, db)
+	ctx := context.Background()
+	s := newReceiptServer(db, nil)
+	text := "unclaimed body"
+	sum := sha256.Sum256([]byte(text))
+	if _, err := db.UpsertDocument(ctx, &store.Document{URL: "https://example.com/mine", Title: "Guide", Text: text, ContentSHA: sum[:]}); err != nil {
+		t.Fatal(err)
+	}
+	receipt := func(final, hash string) crawler.ContributionReceipt {
+		return crawler.ContributionReceipt{FinalURL: final, ContentHash: hash}
+	}
+	good := hex.EncodeToString(sum[:])
+	cancelled, cancel := context.WithCancel(ctx)
+	cancel()
+
+	for name, tc := range map[string]struct {
+		ctx     context.Context
+		raw     string
+		receipt crawler.ContributionReceipt
+		want    bool
+		wantErr bool
+	}{
+		"sole holder":                   {ctx, "https://example.com/mine", receipt("", good), true, false},
+		"the final URL wins over raw":   {ctx, "https://example.com/other", receipt("https://example.com/mine", good), true, false},
+		"unstored document":             {ctx, "https://example.com/missing", receipt("", good), false, false},
+		"content hash is not hex":       {ctx, "https://example.com/mine", receipt("", "not-hex"), false, false},
+		"raw URL that cannot be judged": {ctx, "://not a url", receipt("", good), false, true},
+		"cancelled context":             {cancelled, "https://example.com/mine", receipt("", good), false, true},
+	} {
+		t.Run(name, func(t *testing.T) {
+			got, err := s.contentUnclaimed(tc.ctx, tc.raw, tc.receipt)
+			if got != tc.want || (err != nil) != tc.wantErr {
+				t.Fatalf("contentUnclaimed = %v, %v; want %v, err=%v", got, err, tc.want, tc.wantErr)
+			}
+		})
+	}
+}

cmd/cosift/serve_v1_ceiling_failure_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/cockroachdb/pebble"
+)
+
+func TestModerationReservationRefusesWithoutACounter(t *testing.T) {
+	s := &pebbleHTTP{}
+	err := s.reserveCommunityModeration()
+	var ceiling *ceilingError
+	if err == nil || errors.As(err, &ceiling) {
+		t.Fatalf("a missing store must fail as an error, not as the daily ceiling: %v", err)
+	}
+}
+
+func TestModerationReservationFailsOnACorruptCounterWithoutSpendingTheAllowance(t *testing.T) {
+	s, _ := ceilingServer(t, 5, nil)
+	if err := s.store.DB().Set(communityModerationsKey, []byte("not json"), pebble.Sync); err != nil {
+		t.Fatal(err)
+	}
+	err := s.reserveCommunityModeration()
+	var ceiling *ceilingError
+	if err == nil || errors.As(err, &ceiling) {
+		t.Fatalf("corrupt counter: %v", err)
+	}
+	got, closer, gerr := s.store.DB().Get(communityModerationsKey)
+	if gerr != nil {
+		t.Fatal(gerr)
+	}
+	defer closer.Close()
+	if string(got) != "not json" {
+		t.Fatalf("the counter was overwritten: %q", got)
+	}
+}

internal/community/articles_test.go

@@ -60,3 +60,16 @@ func TestEngineArticlesClassifyTheEnginesAnswers(t *testing.T) {
 		t.Fatal("a non-ULID id reached the engine")
 	}
 }
+
+func TestEngineArticlesReportAnUnreachableEngineWithoutItsAddress(t *testing.T) {
+	engine := httptest.NewServer(http.NotFoundHandler())
+	client, err := newEngineClient(engine.URL, wikiTestKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	engine.Close()
+	_, err = engineArticles{client}.ArticleByID(context.Background(), testRef)
+	if err == nil || errors.Is(err, ErrArticleGone) || strings.Contains(err.Error(), engine.URL) {
+		t.Fatalf("unreachable engine: %v", err)
+	}
+}