diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index e742097778..16789ed7a3 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -208,6 +208,7 @@ jobs: go mod verify go mod tidy -diff GOOS=darwin go vet ./... + go test ./... # go tool would build staticcheck for GOOS too, and Linux cannot run a darwin binary. go build -modfile=tools.mod -o "$RUNNER_TEMP/staticcheck" honnef.co/go/tools/cmd/staticcheck GOOS=darwin "$RUNNER_TEMP/staticcheck" ./... diff --git a/apple/Native/HanaBridge/internal/hana/bridge.go b/apple/Native/HanaBridge/internal/hana/bridge.go index 64ba0bbf4d..43a6002574 100644 --- a/apple/Native/HanaBridge/internal/hana/bridge.go +++ b/apple/Native/HanaBridge/internal/hana/bridge.go @@ -28,6 +28,21 @@ func (b *Bridge) Explain(sessionID uint64, operationID uint64, requestJSON []byt return encoded, encodedFailure(bridgeFailure) } +func (b *Bridge) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) { + encoded, bridgeFailure := b.openCursorOnSession(sessionID, operationID, requestJSON) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.fetchOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.closeCursorOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + func (b *Bridge) Ping(sessionID uint64, operationID uint64) (failure []byte) { return encodedFailure(b.pingSession(sessionID, operationID)) } @@ -101,6 +116,34 @@ func (b *Bridge) explainOnSession(sessionID uint64, operationID uint64, requestJ return entry.explain(operationID, request) } +func (b *Bridge) openCursorOnSession(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + request, failure := decodeRequest[cursorRequest](requestJSON) + if failure != nil { + return nil, failure + } + return entry.openCursor(operationID, request) +} + +func (b *Bridge) fetchOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.fetch(operationID) +} + +func (b *Bridge) closeCursorOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.closeCursor(operationID) +} + func (b *Bridge) pingSession(sessionID uint64, operationID uint64) *bridgeError { entry, failure := b.sessions.lookup(sessionID) if failure != nil { diff --git a/apple/Native/HanaBridge/internal/hana/cursor.go b/apple/Native/HanaBridge/internal/hana/cursor.go new file mode 100644 index 0000000000..949f2b9667 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor.go @@ -0,0 +1,191 @@ +package hana + +import ( + "context" + "database/sql" + "sync/atomic" +) + +const ( + defaultCursorBatchRows = 1_000 + maxCursorBatchRows = 100_000 + cursorBatchBytes = 4 << 20 + noCursorMessage = "no cursor is open for this operation" + cursorParametersMessage = "a cursor takes no parameters" +) + +type cursorRequest struct { + executeRequest + BatchRows int `json:"batchRows"` +} + +func (r cursorRequest) batchRows() int { + if r.BatchRows <= 0 { + return defaultCursorBatchRows + } + return min(r.BatchRows, maxCursorBatchRows) +} + +type cursor struct { + op *operation + rows *sql.Rows + reader *resultReader + maxRows int + maxBytes int +} + +func (c *cursor) nextBatch() (*rowBatch, error) { + batch := &rowBatch{maxRows: c.maxRows, maxBytes: c.maxBytes} + ended, err := c.reader.read(batch) + batch.done = ended + return batch, err +} + +type openedStatement struct { + header *resultEnvelope + rows *sql.Rows + columns []columnInfo +} + +func (s *session) openCursor(operationID uint64, request cursorRequest) ([]byte, *bridgeError) { + if request.hasParameters() { + return nil, internalError(cursorParametersMessage) + } + // Once the result exists a stop sends no CANCEL SESSION: it would mostly reach an idle session, and a refused + // cancel severs the session. The row loop polls the stop flag, and the forced sever covers an unanswered fetch. + var resultReady atomic.Bool + op, conn, failure := s.beginOperation(operationID, func(reason stopReason) { + if !resultReady.Load() { + s.interruptStatement(reason) + } + }) + if failure != nil { + return nil, failure + } + op.stopAfter(secondsDuration(request.TimeoutSeconds)) + opened, err := openStatement(conn, op, request.SQL) + if err != nil { + return nil, s.endOperation(op, err) + } + if opened.rows == nil { + s.slot.finish(op) + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, &rowBatch{done: true}), nil + } + resultReady.Store(true) + // The query timeout covers producing the result. Moving its rows can take far longer. + op.disarmDeadline() + current := &cursor{ + op: op, + rows: opened.rows, + reader: &resultReader{rows: opened.rows, columns: opened.columns, op: op}, + maxRows: request.batchRows(), + maxBytes: cursorBatchBytes, + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, batch), nil +} + +func (s *session) fetch(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + return appendCursorBatchJSON(nil, batch, s.currentState() == sessionLost), nil +} + +func (s *session) closeCursor(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + _, err := runUnderWatchdog(cancelDeadline, s.loseConnection, current.rows.Close) + if isConnectionFailure(err) { + s.loseConnection() + } + s.slot.finish(current.op) + return appendCursorClosedJSON(nil, s.currentState() == sessionLost), nil +} + +func (s *session) advance(current *cursor) (*rowBatch, *bridgeError) { + batch, err := current.nextBatch() + if err == nil && !batch.done { + if s.park(current) { + return batch, nil + } + err = errSlotClosed + } + err = joinCleanupFailure(err, current.rows.Close()) + if failure := s.endOperation(current.op, err); failure != nil { + return nil, failure + } + return batch, nil +} + +func (s *session) endOperation(op *operation, err error) *bridgeError { + op.settle() + var failure *bridgeError + if err != nil { + failure = s.failure(op, err) + } + s.slot.finish(op) + return failure +} + +func (s *session) takeCursor(operationID uint64) (*cursor, *bridgeError) { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return nil, closedError() + } + current := s.parkedCursor + if current == nil || current.op.id != operationID { + return nil, internalError(noCursorMessage) + } + s.parkedCursor = nil + return current, nil +} + +func (s *session) park(current *cursor) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return false + } + s.parkedCursor = current + return true +} + +func openStatement(conn *sql.Conn, op *operation, statement string) (openedStatement, error) { + if op.stopped() { + return openedStatement{}, errOperationStopped + } + if routesToExec(statement) { + envelope, err := runCountingStatement(conn, statement) + return openedStatement{header: envelope}, err + } + rows, err := conn.QueryContext(context.Background(), statement) + if err != nil { + return openedStatement{}, err + } + found, err := moveToFirstResultSet(rows) + var columns []columnInfo + if err == nil && found { + columns, err = describeColumns(rows) + } + if err != nil || !found { + if err = joinCleanupFailure(err, rows.Close()); err != nil { + return openedStatement{}, err + } + return openedStatement{header: affectedRowsEnvelope(0)}, nil + } + return openedStatement{header: tabularEnvelope(columns), rows: rows, columns: columns}, nil +} diff --git a/apple/Native/HanaBridge/internal/hana/cursor_test.go b/apple/Native/HanaBridge/internal/hana/cursor_test.go new file mode 100644 index 0000000000..caea056889 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor_test.go @@ -0,0 +1,412 @@ +package hana + +import ( + "encoding/json" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const selectAll = "SELECT ID, NAME FROM T" + +type cursorReply struct { + Columns []string `json:"columns"` + ColumnTypeNames []string `json:"columnTypeNames"` + RowsAffected int64 `json:"rowsAffected"` + HasResultSet bool `json:"hasResultSet"` + Rows [][]*string `json:"rows"` + TruncatedLobCount int `json:"truncatedLobCount"` + Done bool `json:"done"` + SessionLost bool `json:"sessionLost"` + IsTruncated bool `json:"isTruncated"` +} + +func decodeReply(t *testing.T, encoded []byte, failure *bridgeError) cursorReply { + t.Helper() + if failure != nil { + t.Fatalf("the helper answered with a failure: %v", failure) + } + var reply cursorReply + if err := json.Unmarshal(encoded, &reply); err != nil { + t.Fatalf("the reply is not JSON: %v\n%s", err, encoded) + } + return reply +} + +func requireRefused(t *testing.T, encoded []byte, failure *bridgeError, kind string) *bridgeError { + t.Helper() + if failure == nil { + t.Fatalf("the helper answered %s, want a %s failure", encoded, kind) + } + if failure.Kind != kind { + t.Fatalf("the failure is %v, want kind %s", failure, kind) + } + return failure +} + +func cursorOver(sql string, batchRows int) cursorRequest { + return cursorRequest{executeRequest: executeRequest{SQL: sql}, BatchRows: batchRows} +} + +func mustOpen(t *testing.T, entry *session, operationID uint64, request cursorRequest) cursorReply { + t.Helper() + encoded, failure := entry.openCursor(operationID, request) + return decodeReply(t, encoded, failure) +} + +func mustFetch(t *testing.T, entry *session, operationID uint64) cursorReply { + t.Helper() + encoded, failure := entry.fetch(operationID) + return decodeReply(t, encoded, failure) +} + +func mustExecute(t *testing.T, entry *session, operationID uint64, request executeRequest) cursorReply { + t.Helper() + encoded, failure := entry.execute(operationID, request) + return decodeReply(t, encoded, failure) +} + +func requireFetchRefused(t *testing.T, entry *session, operationID uint64) *bridgeError { + t.Helper() + encoded, failure := entry.fetch(operationID) + return requireRefused(t, encoded, failure, kindInternal) +} + +func parkedOperation(entry *session) *operation { + entry.mu.Lock() + defer entry.mu.Unlock() + return entry.parkedCursor.op +} + +func firstColumn(rows [][]*string) []string { + values := make([]string, 0, len(rows)) + for _, row := range rows { + if row[0] == nil { + values = append(values, "NULL") + continue + } + values = append(values, *row[0]) + } + return values +} + +func numbered(count int) []string { + values := make([]string, count) + for index := range values { + values[index] = strconv.Itoa(index) + } + return values +} + +func drain(t *testing.T, entry *session, operationID uint64, opened cursorReply) (batchSizes []int, ids []string) { + t.Helper() + reply := opened + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + for !reply.Done { + reply = mustFetch(t, entry, operationID) + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + } + return batchSizes, ids +} + +func TestCursorReadsEveryRowInBatchesAndMarksTheLastDone(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + if !slices.Equal(opened.ColumnTypeNames, []string{"INTEGER", "NVARCHAR"}) { + t.Fatalf("column types are %v", opened.ColumnTypeNames) + } + if !opened.HasResultSet || opened.Done { + t.Fatalf("the opened cursor reads hasResultSet %v, done %v", opened.HasResultSet, opened.Done) + } + sizes, ids := drain(t, entry, 1, opened) + if !slices.Equal(sizes, []int{3, 3, 3, 1}) { + t.Fatalf("batch sizes are %v, want [3 3 3 1]", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } + if !server.lastRows().closed.Load() { + t.Fatal("the rows stayed open after the last batch") + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestCursorMarksAnExactMultipleDoneOnItsLastBatch(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(6))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + if opened.Done { + t.Fatal("the first of two full batches reads done") + } + last := mustFetch(t, entry, 1) + + if len(last.Rows) != 3 || !last.Done { + t.Fatalf("the last batch has %d rows and done %v, want 3 and true", len(last.Rows), last.Done) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorOverNoRowsSendsTheHeaderAndDone(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(0))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if len(opened.Rows) != 0 || !opened.Done || !opened.HasResultSet { + t.Fatalf("an empty result opened with %d rows, done %v, hasResultSet %v", len(opened.Rows), opened.Done, opened.HasResultSet) + } + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorEndsABatchAtItsByteBudget(t *testing.T) { + wide := table(10) + wide.name = strings.Repeat("x", 1<<20) + entry := connectedSession(t, newFakeServer(wide)) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 1_000)) + sizes, ids := drain(t, entry, 1, opened) + + if !slices.Equal(sizes, []int{4, 4, 2}) { + t.Fatalf("batch sizes are %v, want [4 4 2] for 1 MiB rows under a 4 MiB budget", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } +} + +func TestStatementWithoutAResultSetOpensNoCursor(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + opened := mustOpen(t, entry, 1, cursorOver("INSERT INTO T VALUES (1, 'a')", 3)) + + if opened.HasResultSet || !opened.Done || opened.RowsAffected != 1 || len(opened.Rows) != 0 { + t.Fatalf("an insert opened as %+v", opened) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestClosingACursorEarlyClosesItsRowsAndFreesTheSlot(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.closeCursor(1) + closed := decodeReply(t, encoded, failure) + + if closed.SessionLost { + t.Fatal("closing a cursor lost the session") + } + if !rows.closed.Load() { + t.Fatal("closing the cursor left its rows open") + } + if served := rows.served.Load(); served != 4 { + t.Fatalf("the server handed out %d rows, want the 3 sent and the 1 read ahead", served) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestStopBetweenFetchesFailsTheNextFetchWithoutCancellingOnTheServer(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + + entry.cancel(1) + op.inFlight.Wait() + encoded, failure := entry.fetch(1) + + requireRefused(t, encoded, failure, kindCancelled) + if !rows.closed.Load() { + t.Fatal("a stopped cursor left its rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop between fetches sent %q, though no statement was running", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestStopDuringAFetchEndsItAtTheNextRowWithoutCancellingOnTheServer(t *testing.T) { + held := table(10) + held.hold = holdRow(4) + server := newFakeServer(held) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + type outcome struct { + encoded []byte + failure *bridgeError + } + fetched := make(chan outcome, 1) + go func() { + encoded, failure := entry.fetch(1) + fetched <- outcome{encoded: encoded, failure: failure} + }() + select { + case <-held.hold.reached: + case <-time.After(5 * time.Second): + t.Fatal("the fetch never reached the held row") + } + + entry.cancel(1) + op.inFlight.Wait() + close(held.hold.released) + + var result outcome + select { + case result = <-fetched: + case <-time.After(5 * time.Second): + t.Fatal("the stopped fetch never answered") + } + requireRefused(t, result.encoded, result.failure, kindCancelled) + if served := rows.served.Load(); served != 5 { + t.Fatalf("the server handed out %d rows, want the fetch to stop at the row it was reading", served) + } + if !rows.closed.Load() { + t.Fatal("a fetch stopped in flight left the rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop after the result existed sent %q", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestFetchThatFailsClosesTheCursor(t *testing.T) { + failing := table(10) + failing.failAt = 5 + server := newFakeServer(failing) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.fetch(1) + + refused := requireRefused(t, encoded, failure, kindInternal) + if !strings.Contains(refused.Message, errFakeRead.Error()) { + t.Fatalf("the failure reads %q, want the server's read error", refused.Message) + } + if !rows.closed.Load() { + t.Fatal("a failed fetch left the rows open") + } + requireFetchRefused(t, entry, 1) + // The next statement reads a table that no longer fails, so only a held slot can fail it. + server.mu.Lock() + server.table.failAt = -1 + server.mu.Unlock() + requireSlotFree(t, entry, 2) +} + +func TestFetchWithNoOpenCursorIsRefused(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + refused := requireFetchRefused(t, entry, 1) + if refused.Message != noCursorMessage { + t.Fatalf("the refusal reads %q", refused.Message) + } + encoded, failure := entry.closeCursor(1) + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestFetchForAnotherOperationLeavesTheCursorOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + mustOpen(t, entry, 5, cursorOver(selectAll, 3)) + + requireFetchRefused(t, entry, 6) + + next := mustFetch(t, entry, 5) + if !slices.Equal(firstColumn(next.Rows), []string{"3", "4", "5"}) { + t.Fatalf("the cursor resumed at %v", firstColumn(next.Rows)) + } +} + +func TestCursorRefusesParameters(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll+" WHERE ID = ?", 3) + request.Parameters = &[]cell{textCell("1")} + + encoded, failure := entry.openCursor(1, request) + + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestQueryTimeoutStopsCoveringTheCursorOnceItIsOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll, 3) + request.TimeoutSeconds = 0.2 + + mustOpen(t, entry, 1, request) + + op := parkedOperation(entry) + op.mu.Lock() + armed := op.deadline != nil + op.mu.Unlock() + if armed { + t.Fatal("the query timeout is still armed after the cursor opened") + } + // Twice the timeout: a deadline left armed would have stopped the cursor by now. + time.Sleep(400 * time.Millisecond) + if op.stopped() { + t.Fatal("the query timeout stopped an open cursor") + } + if next := mustFetch(t, entry, 1); len(next.Rows) != 3 { + t.Fatalf("a fetch past the timeout read %d rows", len(next.Rows)) + } +} + +func TestClosingTheSessionClosesAParkedCursor(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + entry.close() + + deadline := time.Now().Add(5 * time.Second) + for !rows.closed.Load() { + if time.Now().After(deadline) { + t.Fatal("closing the session left the parked cursor's rows open") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestCollectedExecuteStillCutsAtTheRowCap(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + cut := mustExecute(t, entry, 1, executeRequest{SQL: selectAll, RowCap: 4}) + whole := mustExecute(t, entry, 2, executeRequest{SQL: selectAll, RowCap: 10}) + uncapped := mustExecute(t, entry, 3, executeRequest{SQL: selectAll}) + + if !slices.Equal(firstColumn(cut.Rows), numbered(4)) || !cut.IsTruncated { + t.Fatalf("a cap of 4 read %v, truncated %v", firstColumn(cut.Rows), cut.IsTruncated) + } + if len(whole.Rows) != 10 || whole.IsTruncated { + t.Fatalf("a cap equal to the row count read %d rows, truncated %v", len(whole.Rows), whole.IsTruncated) + } + if len(uncapped.Rows) != 10 || uncapped.IsTruncated { + t.Fatalf("no cap read %d rows, truncated %v", len(uncapped.Rows), uncapped.IsTruncated) + } +} diff --git a/apple/Native/HanaBridge/internal/hana/envelope.go b/apple/Native/HanaBridge/internal/hana/envelope.go index 5b5d3a743b..39210549fd 100644 --- a/apple/Native/HanaBridge/internal/hana/envelope.go +++ b/apple/Native/HanaBridge/internal/hana/envelope.go @@ -154,12 +154,8 @@ func tabularEnvelope(columns []columnInfo) *resultEnvelope { } func (e *resultEnvelope) appendJSON(buffer []byte) []byte { - buffer = append(buffer, `{"columns":`...) - buffer = appendStringArray(buffer, e.columns) - buffer = append(buffer, `,"columnTypeNames":`...) - buffer = appendStringArray(buffer, e.columnTypeNames) - buffer = append(buffer, `,"columnClassifications":`...) - buffer = appendOptionalStringArray(buffer, e.columnClassifications) + buffer = append(buffer, '{') + buffer = e.appendColumnFields(buffer) buffer = append(buffer, `,"rows":[`...) for index, row := range e.rows { if index > 0 { @@ -182,6 +178,72 @@ func (e *resultEnvelope) appendJSON(buffer []byte) []byte { return append(buffer, '}') } +func (e *resultEnvelope) appendColumnFields(buffer []byte) []byte { + buffer = append(buffer, `"columns":`...) + buffer = appendStringArray(buffer, e.columns) + buffer = append(buffer, `,"columnTypeNames":`...) + buffer = appendStringArray(buffer, e.columnTypeNames) + buffer = append(buffer, `,"columnClassifications":`...) + return appendOptionalStringArray(buffer, e.columnClassifications) +} + +type rowBatch struct { + maxRows int + maxBytes int + encodedRows []byte + count int + truncatedLobs int + done bool +} + +func (b *rowBatch) full() bool { + return b.count >= b.maxRows || len(b.encodedRows) >= b.maxBytes +} + +func (b *rowBatch) add(row []cell, truncatedLobs int) { + if b.count > 0 { + b.encodedRows = append(b.encodedRows, ',') + } + b.encodedRows = appendRow(b.encodedRows, row) + b.count++ + b.truncatedLobs += truncatedLobs +} + +func appendCursorOpenedJSON(buffer []byte, header *resultEnvelope, batch *rowBatch) []byte { + buffer = append(buffer, '{') + buffer = header.appendColumnFields(buffer) + buffer = append(buffer, `,"rowsAffected":`...) + buffer = strconv.AppendInt(buffer, header.rowsAffected, 10) + buffer = append(buffer, `,"hasResultSet":`...) + buffer = strconv.AppendBool(buffer, header.hasResultSet) + buffer = append(buffer, ',') + buffer = appendBatchFields(buffer, batch, header.sessionLost) + return append(buffer, '}') +} + +func appendCursorBatchJSON(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, '{') + buffer = appendBatchFields(buffer, batch, sessionLost) + return append(buffer, '}') +} + +func appendBatchFields(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, `"rows":[`...) + buffer = append(buffer, batch.encodedRows...) + buffer = append(buffer, `],"truncatedLobCount":`...) + buffer = strconv.AppendInt(buffer, int64(batch.truncatedLobs), 10) + buffer = append(buffer, `,"done":`...) + buffer = strconv.AppendBool(buffer, batch.done) + buffer = append(buffer, `,"sessionLost":`...) + return strconv.AppendBool(buffer, sessionLost) +} + +func appendCursorClosedJSON(buffer []byte, sessionLost bool) []byte { + buffer = append(buffer, `{"sessionLost":`...) + buffer = strconv.AppendBool(buffer, sessionLost) + return append(buffer, '}') +} + func appendRow(buffer []byte, row []cell) []byte { buffer = append(buffer, '[') for index, value := range row { diff --git a/apple/Native/HanaBridge/internal/hana/fakedriver_test.go b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go new file mode 100644 index 0000000000..3ce08ef0e0 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go @@ -0,0 +1,212 @@ +package hana + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "io" + "reflect" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +var errFakeRead = errors.New("the fake server failed to read a row") + +type fakeTable struct { + rowCount int + // The row whose read fails, or -1. + failAt int + name string + hold *rowHold +} + +// Holds the read of one row until the test releases it, like a fetch waiting on the server. +type rowHold struct { + at int + reached chan struct{} + released chan struct{} +} + +func holdRow(at int) *rowHold { + return &rowHold{at: at, reached: make(chan struct{}, 1), released: make(chan struct{})} +} + +func table(rowCount int) fakeTable { + return fakeTable{rowCount: rowCount, failAt: -1} +} + +func (t fakeTable) nameOf(index int) string { + if t.name != "" { + return t.name + } + return "row " + strconv.Itoa(index) +} + +type fakeServer struct { + table fakeTable + + mu sync.Mutex + opened []*fakeRows + execs []string +} + +func newFakeServer(table fakeTable) *fakeServer { + return &fakeServer{table: table} +} + +func (f *fakeServer) Connect(context.Context) (driver.Conn, error) { + return &fakeConn{server: f}, nil +} + +func (f *fakeServer) Driver() driver.Driver { + return fakeDriver{} +} + +func (f *fakeServer) open() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + rows := &fakeRows{table: f.table} + f.opened = append(f.opened, rows) + return rows +} + +func (f *fakeServer) lastRows() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.opened) == 0 { + return nil + } + return f.opened[len(f.opened)-1] +} + +func (f *fakeServer) recordExec(statement string) { + f.mu.Lock() + defer f.mu.Unlock() + f.execs = append(f.execs, statement) +} + +func (f *fakeServer) executed() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.execs...) +} + +type fakeDriver struct{} + +func (fakeDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("the fake server opens connections through its connector") +} + +type fakeConn struct { + server *fakeServer +} + +func (c *fakeConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("the fake server prepares no statements") +} + +func (c *fakeConn) Close() error { + return nil +} + +func (c *fakeConn) Begin() (driver.Tx, error) { + return nil, errors.New("the fake server has no transactions") +} + +func (c *fakeConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return c.server.open(), nil +} + +func (c *fakeConn) ExecContext(_ context.Context, statement string, _ []driver.NamedValue) (driver.Result, error) { + c.server.recordExec(statement) + return driver.RowsAffected(1), nil +} + +type fakeRows struct { + table fakeTable + served atomic.Int64 + closed atomic.Bool +} + +func (r *fakeRows) Columns() []string { + return []string{"ID", "NAME"} +} + +func (r *fakeRows) Close() error { + r.closed.Store(true) + return nil +} + +func (r *fakeRows) Next(dest []driver.Value) error { + index := int(r.served.Load()) + if hold := r.table.hold; hold != nil && index == hold.at { + hold.reached <- struct{}{} + <-hold.released + } + if index == r.table.failAt { + return errFakeRead + } + if index >= r.table.rowCount { + return io.EOF + } + dest[0] = int64(index) + dest[1] = r.table.nameOf(index) + r.served.Add(1) + return nil +} + +func (r *fakeRows) ColumnTypeDatabaseTypeName(index int) string { + if index == 0 { + return "INTEGER" + } + return "NVARCHAR" +} + +func (r *fakeRows) ColumnTypeScanType(index int) reflect.Type { + if index == 0 { + return reflect.TypeFor[int64]() + } + return reflect.TypeFor[string]() +} + +func connectedSession(t *testing.T, server *fakeServer) *session { + t.Helper() + db := sql.OpenDB(server) + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("the fake server refused a connection: %v", err) + } + entry := &session{ + dialer: newTrackingDialer(nil, nil), + db: db, + controlDB: sql.OpenDB(server), + state: sessionConnected, + conn: conn, + connectionID: 1, + } + entry.slot = newOperationSlot(ForcedSeverGrace, entry.loseConnection) + t.Cleanup(entry.close) + return entry +} + +// A slot the cursor still held would block this statement forever. +func requireSlotFree(t *testing.T, entry *session, operationID uint64) { + t.Helper() + finished := make(chan *bridgeError, 1) + go func() { + _, failure := entry.execute(operationID, executeRequest{SQL: "SELECT ID, NAME FROM T"}) + finished <- failure + }() + select { + case failure := <-finished: + if failure != nil { + t.Fatalf("a statement after the cursor failed: %v", failure) + } + case <-time.After(5 * time.Second): + t.Fatal("the cursor still holds the operation slot") + } +} diff --git a/apple/Native/HanaBridge/internal/hana/opslot.go b/apple/Native/HanaBridge/internal/hana/opslot.go index 5afce97c6a..174fbfee15 100644 --- a/apple/Native/HanaBridge/internal/hana/opslot.go +++ b/apple/Native/HanaBridge/internal/hana/opslot.go @@ -154,6 +154,16 @@ func (o *operation) stopAfter(timeout time.Duration) { o.deadline = time.AfterFunc(timeout, func() { o.requestStop(stopTimedOut) }) } +// Unlike settle, the operation stays stoppable: a cursor still answers Stop after its timeout is gone. +func (o *operation) disarmDeadline() { + o.mu.Lock() + defer o.mu.Unlock() + if o.deadline != nil { + o.deadline.Stop() + o.deadline = nil + } +} + func (o *operation) settle() { o.mu.Lock() o.settled = true diff --git a/apple/Native/HanaBridge/internal/hana/session.go b/apple/Native/HanaBridge/internal/hana/session.go index 4e900bff2b..4e66ff3639 100644 --- a/apple/Native/HanaBridge/internal/hana/session.go +++ b/apple/Native/HanaBridge/internal/hana/session.go @@ -64,6 +64,8 @@ type session struct { state sessionState conn *sql.Conn connectionID int64 + // Set only between fetches; a fetch takes it out while it reads. + parkedCursor *cursor controlMu sync.Mutex control *sql.Conn @@ -267,7 +269,11 @@ func (s *session) adopt(conn *sql.Conn, connectionID int64) *bridgeError { } func (s *session) beginStatement(operationID uint64) (*operation, *sql.Conn, *bridgeError) { - op, err := s.slot.begin(operationID, s.interruptStatement) + return s.beginOperation(operationID, s.interruptStatement) +} + +func (s *session) beginOperation(operationID uint64, interrupt interruptFunc) (*operation, *sql.Conn, *bridgeError) { + op, err := s.slot.begin(operationID, interrupt) if err != nil { return nil, nil, s.failure(nil, err) } @@ -380,16 +386,23 @@ func (s *session) close() { s.state = sessionClosed conn := s.conn s.conn = nil + parked := s.parkedCursor + s.parkedCursor = nil s.mu.Unlock() s.slot.close() s.dialer.sever() - go s.release(conn) + go s.release(conn, parked) } -func (s *session) release(conn *sql.Conn) { +func (s *session) release(conn *sql.Conn, parked *cursor) { defer func() { _ = recover() }() + // Open rows hold the connection, so it cannot close before them. + if parked != nil { + _ = parked.rows.Close() + s.slot.finish(parked.op) + } if conn != nil { _ = conn.Close() } diff --git a/apple/Native/HanaBridge/internal/hana/statement.go b/apple/Native/HanaBridge/internal/hana/statement.go index a43e53deeb..5265d9176f 100644 --- a/apple/Native/HanaBridge/internal/hana/statement.go +++ b/apple/Native/HanaBridge/internal/hana/statement.go @@ -152,19 +152,27 @@ func rowsAffected(result sql.Result) int64 { } func readFirstResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope, error) { + found, err := moveToFirstResultSet(rows) + if err != nil { + return nil, err + } + if !found { + return affectedRowsEnvelope(0), nil + } + return readResultSet(rows, op, rowLimit) +} + +func moveToFirstResultSet(rows *sql.Rows) (bool, error) { for { columns, err := rows.Columns() if err != nil { - return nil, err + return false, err } if len(columns) > 0 { - return readResultSet(rows, op, rowLimit) + return true, nil } if !rows.NextResultSet() { - if err := rows.Err(); err != nil { - return nil, err - } - return affectedRowsEnvelope(0), nil + return false, rows.Err() } } } @@ -175,25 +183,68 @@ func readResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope return nil, err } envelope := tabularEnvelope(columns) - for rows.Next() { - if op.stopped() { - return nil, errOperationStopped + reader := &resultReader{rows: rows, columns: columns, op: op} + collected := &collectedRows{limit: rowLimit} + ended, err := reader.read(collected) + if err != nil { + return nil, err + } + envelope.rows = collected.rows + envelope.isTruncated = !ended + envelope.truncatedLobCount = collected.truncatedLobs + return envelope, nil +} + +type rowSink interface { + full() bool + add(row []cell, truncatedLobs int) +} + +type resultReader struct { + rows *sql.Rows + columns []columnInfo + op *operation + // Next has moved to a row that no sink has taken yet, so the next read starts with it. + pending bool +} + +// The bool is true once the result set has no rows left. +func (r *resultReader) read(sink rowSink) (bool, error) { + for { + if !r.pending { + if !r.rows.Next() { + return true, r.rows.Err() + } + r.pending = true } - if rowLimit > 0 && len(envelope.rows) >= rowLimit { - envelope.isTruncated = true - break + if r.op.stopped() { + return false, errOperationStopped } - row, truncatedLobs, err := scanRow(rows, columns, op) + if sink.full() { + return false, nil + } + row, truncatedLobs, err := scanRow(r.rows, r.columns, r.op) if err != nil { - return nil, err + return false, err } - envelope.rows = append(envelope.rows, row) - envelope.truncatedLobCount += truncatedLobs - } - if err := rows.Err(); err != nil { - return nil, err + r.pending = false + sink.add(row, truncatedLobs) } - return envelope, nil +} + +type collectedRows struct { + limit int + rows [][]cell + truncatedLobs int +} + +func (c *collectedRows) full() bool { + return c.limit > 0 && len(c.rows) >= c.limit +} + +func (c *collectedRows) add(row []cell, truncatedLobs int) { + c.rows = append(c.rows, row) + c.truncatedLobs += truncatedLobs } func describeColumns(rows *sql.Rows) (columns []columnInfo, err error) { diff --git a/apple/Native/HanaBridge/server.go b/apple/Native/HanaBridge/server.go index 511a302c9c..0645971207 100644 --- a/apple/Native/HanaBridge/server.go +++ b/apple/Native/HanaBridge/server.go @@ -13,16 +13,19 @@ import ( "github.com/TableProApp/TablePro/Native/HanaBridge/internal/hana" ) -const protocolVersion = 1 +const protocolVersion = 2 const ( - opcodeOpen byte = 1 - opcodeConnect byte = 2 - opcodeExecute byte = 3 - opcodeExplain byte = 4 - opcodePing byte = 5 - opcodeCancel byte = 6 - opcodeClose byte = 7 + opcodeOpen byte = 1 + opcodeConnect byte = 2 + opcodeExecute byte = 3 + opcodeExplain byte = 4 + opcodePing byte = 5 + opcodeCancel byte = 6 + opcodeClose byte = 7 + opcodeOpenCursor byte = 8 + opcodeFetch byte = 9 + opcodeCloseCursor byte = 10 ) const ( @@ -46,6 +49,9 @@ type backend interface { Connect(sessionID uint64, operationID uint64) (result []byte, failure []byte) Execute(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) Explain(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) + CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) Ping(sessionID uint64, operationID uint64) (failure []byte) Cancel(sessionID uint64, operationID uint64) Close(sessionID uint64) @@ -119,6 +125,12 @@ func (s *server) dispatch(request frame.Frame) { s.cancelOperation(request) case opcodeClose: s.closeSession(request) + case opcodeOpenCursor: + s.start(request, s.openCursor) + case opcodeFetch: + s.start(request, s.fetch) + case opcodeCloseCursor: + s.start(request, s.closeCursor) default: s.reply(request.ID, nil, hana.InternalFailure(fmt.Sprintf("unknown opcode %d", request.Code))) } @@ -157,6 +169,18 @@ func (s *server) explain(body requestBody) ([]byte, []byte) { return s.backend.Explain(body.Session, body.Operation, body.Request) } +func (s *server) openCursor(body requestBody) ([]byte, []byte) { + return s.backend.OpenCursor(body.Session, body.Operation, body.Request) +} + +func (s *server) fetch(body requestBody) ([]byte, []byte) { + return s.backend.Fetch(body.Session, body.Operation) +} + +func (s *server) closeCursor(body requestBody) ([]byte, []byte) { + return s.backend.CloseCursor(body.Session, body.Operation) +} + func (s *server) ping(body requestBody) ([]byte, []byte) { if failure := s.backend.Ping(body.Session, body.Operation); failure != nil { return nil, failure diff --git a/apple/Native/HanaBridge/server_test.go b/apple/Native/HanaBridge/server_test.go new file mode 100644 index 0000000000..4eab00f27c --- /dev/null +++ b/apple/Native/HanaBridge/server_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "io" + "slices" + "sync" + "testing" + + "github.com/TableProApp/TablePro/Native/HanaBridge/internal/frame" +) + +type recordedCall struct { + name string + session uint64 + operation uint64 + request string +} + +type recordingBackend struct { + mu sync.Mutex + calls []recordedCall +} + +func (b *recordingBackend) record(call recordedCall) { + b.mu.Lock() + defer b.mu.Unlock() + b.calls = append(b.calls, call) +} + +func (b *recordingBackend) recorded() []recordedCall { + b.mu.Lock() + defer b.mu.Unlock() + return append([]recordedCall(nil), b.calls...) +} + +func (b *recordingBackend) Open([]byte) (uint64, []byte) { + return 1, nil +} + +func (b *recordingBackend) Connect(uint64, uint64) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Execute(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Explain(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, []byte) { + b.record(recordedCall{name: "openCursor", session: sessionID, operation: operationID, request: string(requestJSON)}) + return []byte(`{"done":false}`), nil +} + +func (b *recordingBackend) Fetch(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "fetch", session: sessionID, operation: operationID}) + return []byte(`{"done":true}`), nil +} + +func (b *recordingBackend) CloseCursor(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "closeCursor", session: sessionID, operation: operationID}) + return nil, []byte(`{"kind":"internal","message":"no cursor"}`) +} + +func (b *recordingBackend) Ping(uint64, uint64) []byte { + return nil +} + +func (b *recordingBackend) Cancel(uint64, uint64) {} + +func (b *recordingBackend) Close(uint64) {} + +func (b *recordingBackend) CloseAll() {} + +func readReply(t *testing.T, replies io.Reader) frame.Frame { + t.Helper() + reply, err := frame.Read(replies, frame.MaxBodyLength) + if err != nil { + t.Fatalf("reading a reply failed: %v", err) + } + return reply +} + +func exchange(t *testing.T, requests io.Writer, replies io.Reader, request frame.Frame) frame.Frame { + t.Helper() + if err := frame.Write(requests, request); err != nil { + t.Fatalf("writing frame %d failed: %v", request.ID, err) + } + reply := readReply(t, replies) + if reply.ID != request.ID { + t.Fatalf("the reply answers frame %d, want %d", reply.ID, request.ID) + } + return reply +} + +func TestServerRoutesCursorOperationsAndAnnouncesProtocolTwo(t *testing.T) { + requestReader, requests := io.Pipe() + replies, replyWriter := io.Pipe() + backend := &recordingBackend{} + exited := make(chan int, 1) + go func() { + exited <- run(requestReader, replyWriter, backend) + _ = replyWriter.Close() + }() + + var greeting struct { + Protocol int `json:"protocol"` + } + if err := json.Unmarshal(readReply(t, replies).Body, &greeting); err != nil || greeting.Protocol != 2 { + t.Fatalf("the greeting names protocol %d (%v), want 2", greeting.Protocol, err) + } + + opened := exchange(t, requests, replies, frame.Frame{ID: 1, Code: opcodeOpenCursor, Body: []byte( + `{"session":4,"operation":9,"request":{"sql":"SELECT 1","batchRows":5}}`, + )}) + fetched := exchange(t, requests, replies, frame.Frame{ID: 2, Code: opcodeFetch, Body: []byte(`{"session":4,"operation":9}`)}) + closed := exchange(t, requests, replies, frame.Frame{ID: 3, Code: opcodeCloseCursor, Body: []byte(`{"session":4,"operation":9}`)}) + _ = requests.Close() + + if opened.Code != statusOK || string(opened.Body) != `{"done":false}` { + t.Fatalf("open answered %d %s", opened.Code, opened.Body) + } + if fetched.Code != statusOK || string(fetched.Body) != `{"done":true}` { + t.Fatalf("fetch answered %d %s", fetched.Code, fetched.Body) + } + if closed.Code != statusError { + t.Fatalf("a refused close answered status %d", closed.Code) + } + if code := <-exited; code != exitInputClosed { + t.Fatalf("the server exited with %d", code) + } + want := []recordedCall{ + {name: "openCursor", session: 4, operation: 9, request: `{"sql":"SELECT 1","batchRows":5}`}, + {name: "fetch", session: 4, operation: 9}, + {name: "closeCursor", session: 4, operation: 9}, + } + if got := backend.recorded(); !slices.Equal(got, want) { + t.Fatalf("the backend saw %+v, want %+v", got, want) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift index 2d96cde84c..482f9d9b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift @@ -6,43 +6,55 @@ import Foundation enum InfluxCSV { static func records(_ data: Data) -> [[String]] { var reader = Reader() - data.withUnsafeBytes { reader.read($0) } + reader.read(data) return reader.finish() } - private struct Reader { - var records: [[String]] = [] - var record: [String] = [] - var field: [UInt8] = [] - var inQuotes = false - var lineHasContent = false + // Holds its state between reads, so a body can be split anywhere: inside a quoted field, between the two quotes of + // an escaped one, or between the CR and LF of one line end. + struct Reader { + private var records: [[String]] = [] + private var record: [String] = [] + private var field: [UInt8] = [] + private var inQuotes = false + // A quote right after the one that closed a quoted field is an escaped quote inside it. + private var closedQuote = false + // An LF right after the CR that ended a record is the same line end. + private var endedOnCarriageReturn = false + private var lineHasContent = false + + mutating func read(_ data: Data) { + data.withUnsafeBytes { read($0) } + } mutating func read(_ bytes: UnsafeRawBufferPointer) { - var index = 0 - while index < bytes.count { - let byte = bytes[index] - index += 1 + for byte in bytes { + if endedOnCarriageReturn { + endedOnCarriageReturn = false + if byte == Self.lineFeed { continue } + } if inQuotes { - if byte == Self.quote, index < bytes.count, bytes[index] == Self.quote { - field.append(Self.quote) - index += 1 - } else if byte == Self.quote { + if byte == Self.quote { inQuotes = false + closedQuote = true } else { field.append(byte) } continue } + let afterClosingQuote = closedQuote + closedQuote = false switch byte { case Self.quote: + if afterClosingQuote { field.append(Self.quote) } inQuotes = true lineHasContent = true case Self.comma: endField() lineHasContent = true case Self.carriageReturn: - if index < bytes.count, bytes[index] == Self.lineFeed { index += 1 } endRecord() + endedOnCarriageReturn = true case Self.lineFeed: endRecord() default: @@ -52,9 +64,16 @@ enum InfluxCSV { } } + // The records completed so far. One still open stays for the next read. + mutating func drain() -> [[String]] { + let completed = records + records = [] + return completed + } + mutating func finish() -> [[String]] { if lineHasContent || !record.isEmpty { endRecord() } - return records + return drain() } private mutating func endField() { diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift index cdfa81e9cd..d874a4634b 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift @@ -104,6 +104,13 @@ public struct InfluxGridQuery: Sendable, Equatable, Hashable { return marked ? InfluxStatement.marked(statement, language: language) : statement } + // A read that matched nothing comes back with no columns at all, so the grid still gets them from the schema. + public func emptyResult(schema: [InfluxColumn]) -> InfluxResultTable { + let names = projection.isEmpty ? schema.map(\.name) : projection + let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + return InfluxResultTable(columns: names, typeNames: names.map { types[$0] ?? InfluxTypeName.string }, rows: []) + } + // InfluxQL answers `COUNT(*)` with one `count_` per field, the point count only when there is one field. public static func rowCount(from table: InfluxResultTable) -> Int? { guard let row = table.rows.first else { return 0 } diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift new file mode 100644 index 0000000000..c03223406d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift @@ -0,0 +1,166 @@ +import Foundation + +// A grid page or export read as its body arrives. Each batch holds the rows the collected read gives for the same bytes, +// under one header that the measurement's schema types, as `typed(by:)` types the collected table. +public struct InfluxGridStream { + public enum Body: Sendable, Equatable { + // 3.x SQL and InfluxQL: CSV, with the header record on the first record batch only. + case csv + // 2.x InfluxQL with `chunked=true`: one JSON object per line, each in the v1 shape. + case influxQLChunks + } + + static let batchRows = 5_000 + // A wide header shrinks the batch, so one holds about a million cells at any width. + static let batchCells = 1 << 20 + + private let body: Body + private let schema: [InfluxColumn] + private let emptyResult: InfluxResultTable + private var csv = InfluxCSV.Reader() + private var lines = InfluxJSONLines() + private var layout: InfluxResponseDecoder.V3Layout? + private var header: InfluxResultTable? + private var pending: [[String?]] = [] + private var bytesRead = 0 + private var rowsRead = 0 + + public init(body: Body, grid: InfluxGridQuery, schema: [InfluxColumn]) { + self.body = body + self.schema = schema + emptyResult = grid.emptyResult(schema: schema) + } + + // Every batch carries the same columns and types, and the first one's are the stream's header. + public mutating func consume(_ chunk: Data) throws -> [InfluxResultTable] { + switch body { + case .csv: + bytesRead += chunk.count + csv.read(chunk) + return try csvBatches(csv.drain()) + case .influxQLChunks: + var batches: [InfluxResultTable] = [] + for line in lines.consume(chunk) { + if let batch = try influxQLBatch(line) { batches.append(batch) } + } + return batches + } + } + + // What is left once the body ends. A stream that has sent nothing gets its header here: from the CSV header record, + // or from the schema when the body named no columns. + public mutating func finish() throws -> [InfluxResultTable] { + var batches: [InfluxResultTable] = [] + switch body { + case .csv: + batches = try csvBatches(csv.finish()) + if let layout, !pending.isEmpty || header == nil { + batches.append(batch(pending, layout: layout)) + pending = [] + } + case .influxQLChunks: + if let line = lines.finish(), let batch = try influxQLBatch(line) { + batches.append(batch) + } + } + if header == nil { + batches.append(fixedHeader(emptyResult)) + } + return batches + } + + private mutating func csvBatches(_ records: [[String]]) throws -> [InfluxResultTable] { + var records = records[...] + if layout == nil, let columns = records.popFirst() { + layout = InfluxResponseDecoder.V3Layout(columns: columns) + } + guard let layout, !records.isEmpty else { return [] } + rowsRead += records.count + try InfluxResponseDecoder.CellBudget.perByte(bytes: bytesRead).check(rows: rowsRead, columns: layout.columns.count) + let size = max(1, min(Self.batchRows, Self.batchCells / max(layout.columns.count, 1))) + var batches: [InfluxResultTable] = [] + for record in records { + pending.append(layout.row(record)) + if pending.count >= size { + batches.append(batch(pending, layout: layout)) + pending.removeAll(keepingCapacity: true) + } + } + return batches + } + + // The first batch fixes the header. A column the schema lacks is typed from that batch's values, where the collected + // read infers it from every row. + private mutating func batch(_ rows: [[String?]], layout: InfluxResponseDecoder.V3Layout) -> InfluxResultTable { + let header = self.header ?? fixedHeader(layout.table(rows).typed(by: schema)) + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func influxQLBatch(_ line: Data) throws -> InfluxResultTable? { + let table = try InfluxResponseDecoder.decodeV1JSON(line) + guard !table.columns.isEmpty else { return nil } + guard let header else { + let typed = table.typed(by: schema) + _ = fixedHeader(typed) + return typed + } + if table.columns == header.columns { + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: table.rows) + } + let position = Dictionary(header.columns.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) + let targets = try table.columns.map { column -> Int in + guard let index = position[column] else { + throw InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), column) + ) + } + return index + } + let rows = table.rows.map { row -> [String?] in + var mapped = [String?](repeating: nil, count: header.columns.count) + for (index, target) in targets.enumerated() where index < row.count { + mapped[target] = row[index] + } + return mapped + } + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func fixedHeader(_ table: InfluxResultTable) -> InfluxResultTable { + if let header { return header } + let fixed = InfluxResultTable(columns: table.columns, typeNames: table.typeNames, rows: []) + header = fixed + return fixed + } +} + +// The lines of a body that arrives in pieces, each line whole however the pieces split it. +struct InfluxJSONLines { + private var partial = Data() + + mutating func consume(_ chunk: Data) -> [Data] { + var lines: [Data] = [] + var start = chunk.startIndex + while let end = chunk[start...].firstIndex(of: Self.lineFeed) { + var line = Data() + swap(&line, &partial) + line.append(chunk[start ..< end]) + if !Self.isBlank(line) { lines.append(line) } + start = chunk.index(after: end) + } + partial.append(chunk[start...]) + return lines + } + + mutating func finish() -> Data? { + var line = Data() + swap(&line, &partial) + return Self.isBlank(line) ? nil : line + } + + private static func isBlank(_ line: Data) -> Bool { + line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D || $0 == 0x0A } + } + + private static let lineFeed = UInt8(ascii: "\n") +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift index e70ee42d85..e254fb5ec0 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift @@ -76,6 +76,8 @@ public enum InfluxCredentials: Sendable, Equatable { public enum InfluxRequests { public static let bucketPageSize = 100 + // 2.x's own default for `chunked=true`. + public static let influxQLChunkSize = 10_000 // 3.x with auth on refuses an anonymous `/ping`. Before the version is known the token goes out as `Token`. public static func ping(credentials: InfluxCredentials, generation: InfluxGeneration? = nil) -> InfluxRequest { @@ -148,13 +150,16 @@ public enum InfluxRequests { InfluxRequest(method: "GET", path: "/api/v2/orgs", headers: authHeaders(credentials, generation: .v2)) } + // `streamed` asks 2.x InfluxQL for chunks: without it the server gathers the whole result in its memory before the + // first byte. 3.x writes CSV per record batch either way. public static func query( _ statement: InfluxStatement, database: String, retentionPolicy: String? = nil, organization: String, generation: InfluxGeneration, - credentials: InfluxCredentials + credentials: InfluxCredentials, + streamed: Bool = false ) throws -> InfluxRequest { var headers = authHeaders(credentials, generation: generation) switch (generation, statement.language) { @@ -190,6 +195,10 @@ public enum InfluxRequests { var form = [InfluxQueryItem("q", statement.text)] if !database.isEmpty { form.append(InfluxQueryItem("db", database)) } if let retentionPolicy, !retentionPolicy.isEmpty { form.append(InfluxQueryItem("rp", retentionPolicy)) } + if streamed { + form.append(InfluxQueryItem("chunked", "true")) + form.append(InfluxQueryItem("chunk_size", String(influxQLChunkSize))) + } return InfluxRequest( method: "POST", path: "/query", diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift index 4d1eb53d11..e9d6c7d518 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift @@ -6,24 +6,38 @@ public enum InfluxResponseDecoder { var records = InfluxCSV.records(data)[...] guard let columns = records.popFirst() else { return .empty } try CellBudget.perByte(of: data).check(rows: records.count, columns: columns.count) - let rows: [[String?]] = records.map { record in - columns.indices.map { index in + let layout = V3Layout(columns: columns) + return layout.table(records.map(layout.row)) + } + + // A 3.x CSV record as a row under its header. The stream maps records with this too, so a cell reads the same. + struct V3Layout { + let columns: [String] + // Only `time` is inferred as a timestamp, so it is the one column whose zone-less values get a `Z`. + private let timeColumns: [Int] + + init(columns: [String]) { + self.columns = columns + timeColumns = columns.indices.filter { columns[$0] == InfluxTypeName.timeColumn } + } + + func row(_ record: [String]) -> [String?] { + var row: [String?] = columns.indices.map { index in guard index < record.count, !record[index].isEmpty else { return nil } return record[index] } - } - let typeNames = columns.indices.map { index in - InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) - } - let timestampColumns = typeNames.indices.filter { typeNames[$0] == InfluxTypeName.timestamp } - let normalized = timestampColumns.isEmpty ? rows : rows.map { row in - var row = row - for index in timestampColumns { + for index in timeColumns { row[index] = row[index].map(InfluxTypeName.normalizedTimestamp) } return row } - return InfluxResultTable(columns: columns, typeNames: typeNames, rows: normalized) + + func table(_ rows: [[String?]]) -> InfluxResultTable { + let typeNames = columns.indices.map { index in + InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) + } } // InfluxQL over `/query` (1.x and the 2.x compatibility API). Every series of every statement lands in one grid: a @@ -159,12 +173,16 @@ public enum InfluxResponseDecoder { } // Rows are as wide as every header together, so a short record costs more cells than bytes. - private struct CellBudget { + struct CellBudget { let limit: Int // Every v3 CSV row carries every column, a byte per cell at least: 16 cells per byte, or a million at any size. static func perByte(of data: Data) -> CellBudget { - let (scaled, overflow) = data.count.multipliedReportingOverflow(by: 16) + perByte(bytes: data.count) + } + + static func perByte(bytes: Int) -> CellBudget { + let (scaled, overflow) = bytes.multipliedReportingOverflow(by: 16) return CellBudget(limit: overflow ? .max : max(scaled, 1 << 20)) } diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift index 21fb9221ce..b3502139a6 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -52,7 +52,10 @@ public enum MemcachedKeyListing { guard let equals = part.firstIndex(of: "=") else { continue } fields[part[.. + // The columns left to their default, by name like `expressionColumns`. Every other cell is a value. + public let defaultColumns: Set public init( rowIndex: Int, kind: Kind, cellChanges: [CellChange] = [], originalRow: [SpannerCell]? = nil, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) { self.rowIndex = rowIndex self.kind = kind self.cellChanges = cellChanges self.originalRow = originalRow self.expressionColumns = expressionColumns + self.defaultColumns = defaultColumns } } public enum SpannerRowEditSQL { - public static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "SpannerRowEditSQL") public static func statements( @@ -115,7 +117,7 @@ internal struct SpannerRowEditTarget { func insert(_ change: SpannerRowChange, row: [SpannerCell]?) -> SpannerRenderedStatement? { let cells = row.map { zip(columns, $0).map { SpannerRowChange.CellChange(column: $0.0, value: $0.1) } } ?? Self.lastValuePerColumn(change.cellChanges) - let written = cells.filter { !Self.isDefaultSentinel($0.value) } + let written = cells.filter { !change.defaultColumns.contains($0.column) } guard written.isEmpty else { return Self.insertStatement( table: table, cells: written, dialect: dialect, expressionColumns: change.expressionColumns @@ -132,7 +134,7 @@ internal struct SpannerRowEditTarget { var parameters: [SpannerCell] = [] let assignments = Self.lastValuePerColumn(change.cellChanges).map { cell in let column = dialect.quoteIdentifier(cell.column) - guard !Self.isDefaultSentinel(cell.value) else { return "\(column) = DEFAULT" } + guard !change.defaultColumns.contains(cell.column) else { return "\(column) = DEFAULT" } if let sql = Self.expression(cell, in: change.expressionColumns) { return "\(column) = \(sql)" } parameters.append(cell.value) return "\(column) = ?" @@ -191,10 +193,6 @@ internal struct SpannerRowEditTarget { return (terms.joined(separator: " AND "), parameters) } - private static func isDefaultSentinel(_ cell: SpannerCell) -> Bool { - cell == .text(SpannerRowEditSQL.defaultSentinel) - } - private static func lastValuePerColumn(_ cells: [SpannerRowChange.CellChange]) -> [SpannerRowChange.CellChange] { var order: [String] = [] var values: [String: SpannerCell] = [:] diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index 440de15bef..8f8d3c83fd 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -31,7 +31,7 @@ public final class TeradataConnection: @unchecked Sendable { private var _stoppingRequest: UInt32? // 3110: "The transaction was aborted by the user." - static let userAbortCode = 3_110 + public static let userAbortCode = 3_110 private static let abortLockWait: TimeInterval = 0.25 // A server or network that stops answering would otherwise hold a stop for the whole 20-second receive timeout. static let defaultStopGrace: TimeInterval = 5 @@ -53,9 +53,11 @@ public final class TeradataConnection: @unchecked Sendable { } private enum Exchange { - case answered([Parcel]) + case answered case notSent case outputEnded + // The consumer threw on a reply, and the request has ended on the wire. + case refused(any Error) } public init(config: TeradataConnectionConfig) { @@ -120,8 +122,7 @@ public final class TeradataConnection: @unchecked Sendable { established = true } - /// Keeps the transport's connect deadline active through the driver's required VERSION probe. - /// Only then may ordinary query I/O use the session's 20-second socket timeout. + // The connect deadline holds through the driver's VERSION probe; only then does query I/O get the 20-second timeout. public func finishConnecting() throws { guard let socket, let connectDeadline else { throw TeradataWireError.connectionFailed("not connecting") @@ -132,14 +133,62 @@ public final class TeradataConnection: @unchecked Sendable { } public func execute(_ sql: String) throws -> TeradataResultSet { + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + rows.append(contentsOf: batch) + return true + }) + return TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount) + } + + // The reply that passes the cap ends the output on the server, so the rows past it are never fetched. + public func execute(_ sql: String, rowCap: Int?) throws -> TeradataCappedResult { + guard let rowCap, rowCap > 0 else { + return TeradataCappedResult(resultSet: try execute(sql), isTruncated: false) + } + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + var isTruncated = false + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + guard rows.count + batch.count > rowCap else { + rows.append(contentsOf: batch) + return true + } + rows.append(contentsOf: batch.prefix(rowCap - rows.count)) + isTruncated = true + return false + }) + return TeradataCappedResult( + resultSet: TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount), + isTruncated: isTruncated) + } + + // `onColumns` runs once before any rows, with none for a request that has no result set; false from `onRows` ends the output. + public func stream( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws { + _ = try perform(sql, onColumns: onColumns, onRows: onRows) + } + + private func perform( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Int { guard socket != nil else { throw TeradataWireError.connectionFailed("not connected") } let requestNumber = nextRequest() let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() + var reader = TeradataResultParser.Reader() let outcome: Exchange do { - outcome = try exchange(body, requestNumber: requestNumber) + outcome = try exchange(body, requestNumber: requestNumber) { parcels in + try reader.read(parcels, onColumns: onColumns, onRows: onRows) + } } catch { endRequest() // The reply still on its way would be read as the next request's. @@ -147,16 +196,28 @@ public final class TeradataConnection: @unchecked Sendable { throw error } let stopped = endRequest() - guard case .answered(let parcels) = outcome else { throw TeradataWireError.cancelled } - do { - return try TeradataResultParser.parse(parcels) - } catch TeradataWireError.server(let code, _) where stopped && code == Self.userAbortCode { - // The server rolled back the transaction the request ran in; the host has no way yet to say so. + switch outcome { + case .notSent: + throw TeradataWireError.cancelled + case .refused(let error): + if stopped, case .server(let code, _)? = error as? TeradataWireError, code == Self.userAbortCode { + // In Teradata mode a request outside BT is its own transaction, so the abort undid that request alone. + throw config.transactionMode == .ansi ? TeradataWireError.rolledBackByStop : TeradataWireError.cancelled + } + throw error + case .outputEnded where stopped: throw TeradataWireError.cancelled + case .answered, .outputEnded: + try reader.finish(onColumns: onColumns) + return reader.activityCount } } - private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> Exchange { + private func exchange( + _ body: [UInt8], + requestNumber: UInt32, + consume: ([Parcel]) throws -> Bool + ) throws -> Exchange { let sent = try wireLock.withLock { () -> Bool in // A stop that landed before the request went out refuses it, and the session stays as it was. let refused = socketLock.withLock { () -> Bool in @@ -169,17 +230,21 @@ public final class TeradataConnection: @unchecked Sendable { return true } guard sent else { return .notSent } - var parcels: [Parcel] = [] while true { let batch = try readReply() - parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { return .answered(parcels) } + let complete = Self.responseIsComplete(batch) // Once the first reply is out an abort no longer takes effect, so a stop ends the output instead. - let stopRequested = socketLock.withLock { () -> Bool in - _phase = .fetching(requestNumber) - return _stopRequested + socketLock.withLock { _phase = .fetching(requestNumber) } + let wantsMore: Bool + do { + wantsMore = try consume(batch) + } catch { + // Ending the output keeps the session; a failed end leaves the rest to be read as the next request's. + if !complete, (try? endOutput(requestNumber: requestNumber)) == nil { throw error } + return .refused(error) } - if stopRequested { + if complete { return .answered } + if !wantsMore || socketLock.withLock({ _stopRequested }) { try endOutput(requestNumber: requestNumber) return .outputEnded } @@ -516,3 +581,9 @@ public final class TeradataConnection: @unchecked Sendable { return String(cString: buffer) } } + +public struct TeradataCappedResult: Sendable { + public let resultSet: TeradataResultSet + // The answer held more rows than the cap, and its output was ended there. + public let isTruncated: Bool +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift index b8c409d28f..13073c15a2 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift @@ -2,36 +2,62 @@ import Foundation import TableProWireIO enum TeradataResultParser { - static func parse(_ parcels: [Parcel]) throws -> TeradataResultSet { - for parcel in parcels where parcel.flavor == ParcelFlavor.failure.rawValue - || parcel.flavor == ParcelFlavor.error.rawValue - || parcel.flavor == ParcelFlavor.statementError.rawValue { - let (code, message) = errorDetail(parcel) - throw TeradataWireError.server(code: code, message: message) - } + // The answer is its first result set; a later statement's rows join only when they decode the same way. + struct Reader { + private var resultColumns: [ColumnMeta]? + private var statementColumns: [ColumnMeta] = [] + private var statementJoins = false + private var firstActivityCount: Int? - let activityCount = parcels - .first { $0.flavor == ParcelFlavor.success.rawValue } - .map { readActivityCount(fromSuccess: $0.body) } ?? 0 + var activityCount: Int { firstActivityCount ?? 0 } - var columns: [ColumnMeta] = [] - if let prepInfoX = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfoX.rawValue }) { - columns = try parsePrepInfo(prepInfoX.body, extended: true) - } else if let prepInfo = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfo.rawValue }) { - columns = try parsePrepInfo(prepInfo.body, extended: false) - } - - var rows: [[TeradataValue]] = [] - if !columns.isEmpty { - for record in parcels where record.flavor == ParcelFlavor.record.rawValue { - rows.append(try RecordDecoder.decode(recordBody: record.body, columns: columns)) + // False when the consumer wants no more rows. + mutating func read( + _ parcels: [Parcel], + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Bool { + if let failure = parcels.first(where: TeradataResultParser.isError) { + let (code, message) = TeradataResultParser.errorDetail(failure) + throw TeradataWireError.server(code: code, message: message) + } + var rows: [[TeradataValue]] = [] + for parcel in parcels { + switch parcel.knownFlavor { + case .success?: + if firstActivityCount == nil { + firstActivityCount = TeradataResultParser.readActivityCount(fromSuccess: parcel.body) + } + case .prepInfo?, .prepInfoX?: + let columns = try TeradataResultParser.parsePrepInfo( + parcel.body, extended: parcel.flavor == ParcelFlavor.prepInfoX.rawValue) + if let resultColumns { + statementJoins = !resultColumns.isEmpty + && TeradataResultParser.decodeAlike(columns, resultColumns) + } else { + resultColumns = columns + statementJoins = !columns.isEmpty + try onColumns(columns.map { + TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + }) + } + statementColumns = columns + case .record? where statementJoins: + rows.append(try RecordDecoder.decode(recordBody: parcel.body, columns: statementColumns)) + default: + break + } } + guard !rows.isEmpty else { return true } + return try onRows(rows) } - let resultColumns = columns.map { - TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + // A request that described no result set still answers its consumer, with no columns. + mutating func finish(onColumns: ([TeradataColumn]) throws -> Void) throws { + guard resultColumns == nil else { return } + resultColumns = [] + try onColumns([]) } - return TeradataResultSet(columns: resultColumns, rows: rows, activityCount: activityCount) } static func errorDetail(_ parcel: Parcel) -> (code: Int, message: String) { @@ -45,6 +71,18 @@ enum TeradataResultParser { return (code, message.isEmpty ? decodedText(body) : message) } + private static func isError(_ parcel: Parcel) -> Bool { + parcel.flavor == ParcelFlavor.failure.rawValue + || parcel.flavor == ParcelFlavor.error.rawValue + || parcel.flavor == ParcelFlavor.statementError.rawValue + } + + // A record decodes by each field's base type and declared length, so two descriptions alike in those read it alike. + private static func decodeAlike(_ lhs: [ColumnMeta], _ rhs: [ColumnMeta]) -> Bool { + lhs.count == rhs.count + && zip(lhs, rhs).allSatisfy { $0.baseCode == $1.baseCode && $0.dataLength == $1.dataLength } + } + private static func decodedText(_ bytes: some Collection) -> String { String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift index a82563f33b..af9f40875e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift @@ -8,6 +8,8 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case server(code: Int, message: String) case unsupported(String) case cancelled + // Stop aborted the request, and the server rolled back the transaction it ran in. + case rolledBackByStop public var description: String { switch self { @@ -17,6 +19,7 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case .server(let code, let message): return "\(message) (\(code))" case .unsupported(let what): return "unsupported: \(what)" case .cancelled: return "cancelled" + case .rolledBackByStop: return "stopped, and the server rolled back the transaction (\(TeradataConnection.userAbortCode))" } } diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift new file mode 100644 index 0000000000..d5ea9e7298 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift @@ -0,0 +1,224 @@ +import Foundation +@testable import TableProInfluxDBCore +import Testing + +private func bytes(_ text: String) -> Data { + Data(text.utf8) +} + +private func pieces(_ data: Data, size: Int) -> [Data] { + stride(from: 0, to: data.count, by: size).map { data.subdata(in: $0 ..< min($0 + size, data.count)) } +} + +private func records(_ chunks: [Data]) -> [[String]] { + var reader = InfluxCSV.Reader() + var records: [[String]] = [] + for chunk in chunks { + reader.read(chunk) + records += reader.drain() + } + return records + reader.finish() +} + +private func streamed( + _ body: InfluxGridStream.Body, + _ chunks: [Data], + grid: InfluxGridQuery = cpuExport, + schema: [InfluxColumn] = cpuSchema +) throws -> [InfluxResultTable] { + var stream = InfluxGridStream(body: body, grid: grid, schema: schema) + var batches: [InfluxResultTable] = [] + for chunk in chunks { + batches += try stream.consume(chunk) + } + let rest = try stream.finish() + return batches + rest +} + +// The batches under the header the first one carries, which every other one repeats. +private func joined(_ batches: [InfluxResultTable]) throws -> InfluxResultTable { + let first = try #require(batches.first) + #expect(batches.allSatisfy { $0.columns == first.columns && $0.typeNames == first.typeNames }) + return InfluxResultTable(columns: first.columns, typeNames: first.typeNames, rows: batches.flatMap { $0.rows }) +} + +private let cpuExport = InfluxGridQuery(measurement: "cpu", shape: .all) + +private let cpuSchema = [ + InfluxColumn(name: "time", typeName: InfluxTypeName.timestamp, role: .time), + InfluxColumn(name: "host", typeName: InfluxTypeName.tag, role: .tag), + InfluxColumn(name: "idle", typeName: InfluxTypeName.integer, role: .field), + InfluxColumn(name: "note", typeName: InfluxTypeName.string, role: .field), + InfluxColumn(name: "usage", typeName: InfluxTypeName.float, role: .field), +] + +private let v3Body = bytes( + "host,time,usage,note,idle\r\n" + + "a,2024-10-27T03:33:20,12.5,\"hot, \"\"quoted\"\"\",10\r\n" + + "b,2024-10-27T03:34:20.500Z,,,\r\n" + + ",2024-10-27T03:35:20,7,\"two\r\nlines é\",40\r\n" +) + +private let v2Rows = [ + #"["2024-10-27T03:35:20Z","c",10,"a;b",7]"#, + #"["2024-10-27T03:34:20Z","b",40,"hot, \"quoted\"",50.1]"#, + #"["2024-10-27T03:33:20Z","a",null,null,12.5]"#, +] + +private func v2Chunk(_ rows: ArraySlice, partial: Bool) -> String { + let flag = partial ? #","partial":true"# : "" + return #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","idle","note","usage"],"values":["# + + rows.joined(separator: ",") + "]" + flag + "}]" + flag + "}]}" +} + +private let tooWide = InfluxError.malformedResponse( + String(localized: "the result has far more columns than its rows fill") +) + +struct InfluxGridStreamTests { + @Test func aCREndingOneChunkAndTheLFStartingTheNextAreOneLineEnd() { + #expect(records([bytes("a,b\r"), bytes("\nc,d\r\n")]) == [["a", "b"], ["c", "d"]]) + } + + @Test func anEscapedQuoteSplitBetweenItsTwoQuotesIsOneQuote() { + #expect(records([bytes("\"x\""), bytes("\"y\",z\n")]) == [["x\"y", "z"]]) + } + + @Test func aQuotedLineEndSplitAcrossChunksStaysInTheField() { + #expect(records([bytes("\"a\r"), bytes("\nb\",c\n"), bytes("\"d\n"), bytes("e\"\n")]) == [["a\r\nb", "c"], ["d\ne"]]) + } + + @Test("Every chunk size reads the records one whole body gives", arguments: [1, 2, 3, 7, 64]) + func chunkSizeDoesNotChangeTheRecords(size: Int) { + var body = bytes("host,note\r\na,\"x, \"\"y\"\"\"\r\n\r\nb,\"two\nlines\"\nc,日本\r\"\"\"\",é\r\n") + body.append(contentsOf: [0x61, 0xFF, 0x62, 0x2C, 0x7A]) + #expect(InfluxCSV.records(body) == [ + ["host", "note"], + ["a", "x, \"y\""], + [""], + ["b", "two\nlines"], + ["c", "日本"], + ["\"", "é"], + ["a\u{FFFD}b", "z"], + ]) + #expect(records(pieces(body, size: size)) == InfluxCSV.records(body)) + } + + @Test("A v3 stream reads the rows and types the collected read gives", arguments: [1, 3, 64, 100_000]) + func aV3StreamReadsLikeTheCollectedRead(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV3CSV(v3Body).typed(by: cpuSchema) + let table = try joined(streamed(.csv, pieces(v3Body, size: size))) + #expect(table == collected) + #expect(table.typeNames == [ + InfluxTypeName.tag, InfluxTypeName.timestamp, InfluxTypeName.float, InfluxTypeName.string, InfluxTypeName.integer, + ]) + #expect(table.column(named: "time") == ["2024-10-27T03:33:20Z", "2024-10-27T03:34:20.500Z", "2024-10-27T03:35:20Z"]) + } + + @Test func aV3StreamSendsBatchesOfFiveThousandRows() throws { + let body = bytes("host,usage\n" + String(repeating: "a,1\n", count: 12_001)) + let batches = try streamed(.csv, [body]) + #expect(batches.map { $0.rows.count } == [5_000, 5_000, 2_001]) + #expect(try joined(batches).typeNames == [InfluxTypeName.tag, InfluxTypeName.float]) + } + + @Test func aWideV3HeaderShrinksTheBatchToAboutAMillionCells() throws { + let columns = (0 ..< 4_096).map { "c\($0)" } + let record = Array(repeating: "1", count: columns.count).joined(separator: ",") + let body = bytes(columns.joined(separator: ",") + "\n" + String(repeating: record + "\n", count: 257)) + let batches = try streamed(.csv, [body], schema: []) + #expect(batches.map { $0.rows.count } == [256, 1]) + } + + @Test("A v3 header of 40,000 columns over 1,000 empty records is refused, as the collected read refuses it") + func shortV3RecordsUnderAWideHeaderAreRefused() { + let header = (0 ..< 40_000).map { "c\($0)" }.joined(separator: ",") + let body = bytes(header + "\n" + String(repeating: "\n", count: 1_000)) + #expect(throws: tooWide) { try streamed(.csv, [body], schema: []) } + } + + @Test func aV3HeaderRecordWithNoRowsKeepsItsColumns() throws { + let body = bytes("usage,time,extra\n") + let batches = try streamed(.csv, [body]) + #expect(batches == [InfluxResultTable( + columns: ["usage", "time", "extra"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + let collected = try InfluxResponseDecoder.decodeV3CSV(body).typed(by: cpuSchema) + #expect(batches == [collected]) + } + + @Test func aV3ColumnTheSchemaLacksIsTypedFromTheFirstBatch() throws { + let table = try joined(streamed(.csv, [bytes("time,extra\n2024-10-27T03:33:20Z,1\n2024-10-27T03:34:20Z,2\n")])) + #expect(table.typeNames == [InfluxTypeName.timestamp, InfluxTypeName.integer]) + } + + @Test("An empty body gets the header from the schema", arguments: [InfluxGridStream.Body.csv, .influxQLChunks]) + func anEmptyBodyGetsTheHeaderFromTheSchema(body: InfluxGridStream.Body) throws { + let empty = body == .csv ? Data() : bytes(#"{"results":[{"statement_id":0}]}"# + "\n") + #expect(try streamed(body, [empty]) == [InfluxResultTable( + columns: ["time", "host", "idle", "note", "usage"], + typeNames: [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ], + rows: [] + )]) + let page = InfluxGridQuery(measurement: "cpu", projection: ["usage", "time", "gone"], shape: .page(limit: 50, offset: 0)) + #expect(try streamed(body, [empty], grid: page) == [InfluxResultTable( + columns: ["usage", "time", "gone"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + } + + @Test("v2 chunks read the rows and types one collected body gives", arguments: [1, 5, 64, 100_000]) + func v2ChunksReadLikeTheCollectedBody(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV1JSON(bytes(v2Chunk(v2Rows[...], partial: false))) + .typed(by: cpuSchema) + let body = bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n" + v2Chunk(v2Rows[2...], partial: false) + "\n") + let batches = try streamed(.influxQLChunks, pieces(body, size: size)) + #expect(batches.map { $0.rows.count } == [2, 1]) + #expect(try joined(batches) == collected) + #expect(collected.typeNames == [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ]) + } + + @Test func aLastV2ChunkWithoutItsNewlineIsStillRead() throws { + let batches = try streamed(.influxQLChunks, [bytes(v2Chunk(v2Rows[...], partial: false))]) + #expect(batches.map { $0.rows.count } == [3]) + } + + @Test( + "An error object after some rows throws", + arguments: [ + #"{"results":[{"statement_id":0,"error":"max-select-point limit exceeded"}]}"#, + #"{"error":"max-select-point limit exceeded"}"#, + ] + ) + func anErrorObjectAfterSomeRowsThrows(errorLine: String) throws { + var stream = InfluxGridStream(body: .influxQLChunks, grid: cpuExport, schema: cpuSchema) + let first = try stream.consume(bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n")) + #expect(first.map { $0.rows.count } == [2]) + #expect(throws: InfluxError.query("max-select-point limit exceeded")) { + try stream.consume(bytes(errorLine + "\n")) + } + } + + @Test func aLaterV2ChunkWithItsColumnsReorderedMapsThemByName() throws { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["usage","time"],"values":[[1.5,"2024-10-27T03:30:00Z"]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let table = try joined(streamed(.influxQLChunks, [body])) + #expect(table.rows.last == ["2024-10-27T03:30:00Z", nil, nil, nil, "1.5"]) + } + + @Test func aLaterV2ChunkWithAColumnTheFirstLacksThrows() { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","extra"],"values":[["2024-10-27T03:30:00Z",1]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let refusal = InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), "extra") + ) + #expect(throws: refusal) { try streamed(.influxQLChunks, [body]) } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift index 6a032a6746..8aae352353 100644 --- a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift @@ -224,6 +224,38 @@ struct InfluxRequestTests { #expect(try InfluxCatalogDecoder.v3Databases(body) == ["_internal", "metrics"]) } + @Test func aStreamedV2InfluxQLQueryAsksForChunksOfTenThousand() throws { + let statement = InfluxStatement(language: .influxQL, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token, streamed: true + ) + #expect(queryItems(ofForm: streamed.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + URLQueryItem(name: "chunked", value: "true"), + URLQueryItem(name: "chunk_size", value: "10000"), + ]) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token + ) + #expect(queryItems(ofForm: collected.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + ]) + } + + @Test(arguments: [InfluxQueryLanguage.sql, .influxQL]) + func aStreamedV3QueryIsTheCollectedRequest(language: InfluxQueryLanguage) throws { + let statement = InfluxStatement(language: language, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token, streamed: true + ) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token + ) + #expect(streamed == collected) + } + @Test func anInfluxQLFormBodyCarriesTheRetentionPolicy() throws { let request = try InfluxRequests.query( InfluxStatement(language: .influxQL, text: "SHOW MEASUREMENTS"), diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift index 62c2c55cf8..6ee0c64ead 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -12,6 +12,14 @@ struct MemcachedKeyListingTests { #expect(entry?.secondsLeft(at: 1_793_499_867) == 300) } + @Test("A listed key a text command line cannot carry is left out", arguments: [ + "key=x%0D%0Aflush_all exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + "key=two%20words exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + ]) + func unsendableKeyIsLeftOut(line: String) { + #expect(MemcachedKeyListing.metadumpEntry(line) == nil) + } + @Test("exp=-1 means the item never expires") func metadumpNeverExpires() { let entry = MemcachedKeyListing.metadumpEntry("key=%22sp%22 exp=-1 la=1 cas=4 fetch=no cls=1 size=64 flags=0") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift new file mode 100644 index 0000000000..2d29b5378b --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift @@ -0,0 +1,189 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginTypedParameterBindingTests { + private func bound( + _ values: [String], + kinds: [PluginColumnKind?]?, + query: String = "SELECT * FROM t WHERE a = ?" + ) async throws -> String { + let driver = RecordingDriver() + var context = PluginStatementContext() + context.parameterKinds = kinds + _ = try await driver.executeParameterized(query: query, parameters: values.map { .text($0) }, context: context) + return try #require(driver.executed.first) + } + + @Test("A String parameter is quoted whatever it looks like", arguments: ["1E5", "42", "00123", "-7"]) + func stringIsQuoted(value: String) async throws { + #expect(try await bound([value], kinds: [.text]) == "SELECT * FROM t WHERE a = '\(value)'") + } + + @Test("An Integer or Decimal parameter that reads as that number is bare") + func numbersAreBare() async throws { + #expect(try await bound(["42"], kinds: [.integer]) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["1E5"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = 1E5") + } + + @Test("Text that is not the declared number is sent as a string for the server to refuse") + func notANumberIsQuoted() async throws { + #expect(try await bound(["1.5"], kinds: [.integer]) == "SELECT * FROM t WHERE a = '1.5'") + #expect(try await bound(["1; DROP TABLE t"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = '1; DROP TABLE t'") + } + + @Test("A Boolean parameter is TRUE or FALSE", arguments: [("true", "TRUE"), ("0", "FALSE"), ("Yes", "TRUE")]) + func booleans(value: String, literal: String) async throws { + #expect(try await bound([value], kinds: [.boolean]) == "SELECT * FROM t WHERE a = \(literal)") + } + + @Test("A Boolean parameter that is no boolean is a string") + func notABooleanIsQuoted() async throws { + #expect(try await bound(["maybe"], kinds: [.boolean]) == "SELECT * FROM t WHERE a = 'maybe'") + } + + @Test("A Date parameter is a quoted string") + func dateIsQuoted() async throws { + #expect(try await bound(["2026-01-01"], kinds: [.other]) == "SELECT * FROM t WHERE a = '2026-01-01'") + } + + @Test("A value with no type keeps the reading by shape") + func untypedKeepsTheShape() async throws { + #expect(try await bound(["42"], kinds: nil) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["42", "x"], kinds: [.text, nil], query: "SELECT ?, ?") == "SELECT '42', 'x'") + #expect(try await bound(["42"], kinds: [nil]) == "SELECT * FROM t WHERE a = 42") + } + + @Test("Each parameter takes its own type by position") + func typesFollowPosition() async throws { + let sql = try await bound(["7", "7"], kinds: [.integer, .text], query: "SELECT ?, ?") + #expect(sql == "SELECT 7, '7'") + } + + @Test("A driver that forwards its context overload keeps the types through withHostContext") + func forwardingDriverKeepsTheTypes() async throws { + let driver = ForwardingDriver() + var context = PluginStatementContext() + context.parameterKinds = [.text] + _ = try await driver.executeUserQuery( + query: "SELECT ?", rowCap: nil, parameters: [.text("5")], context: context + ) + #expect(driver.inner.executed == ["SELECT '5'"]) + } +} + +struct PluginRowChangeDefaultMarkTests { + @Test("Only a marked column is the default") + func onlyMarkedColumnsAreDefaults() { + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + #expect(!change.isColumnDefault(0)) + change.defaultColumns = [1] + #expect(change.isColumnDefault(1)) + #expect(!change.isColumnDefault(0)) + } + + @Test("An engine with no defaults refuses the first change that leaves a cell to one, naming its row") + func defaultsUnsupportedNamesTheRow() { + let plain = PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil) + var defaulted = PluginRowChange(rowIndex: 7, type: .update, cellChanges: [], originalRow: nil) + defaulted.defaultColumns = [0] + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain]) == nil) + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain, defaulted])?.rowIndex == 7) + } +} + +struct PluginBoundedStreamDeclaredLimitTests { + private func stream(rows: Int, rowLimit: Int?) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + var header = PluginStreamHeader(columns: ["id"], columnTypeNames: ["INT"]) + header.rowLimit = rowLimit + continuation.yield(.header(header)) + continuation.yield(.rows((0.. PluginQueryResult { + lock.withLock { statements.append(query) } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +// The shape of a driver that overrides the context overload to set its own state, then runs the context-free one. +private final class ForwardingDriver: PluginDatabaseDriver, @unchecked Sendable { + let inner = RecordingDriver() + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { + try await inner.execute(query: query) + } + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + try await PluginParameterBinder.withHostContext(context) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index cd0bbbdaab..279cc22ae3 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -98,14 +98,22 @@ struct QdrantRowWriteTests { #expect(try written(#"["a"]"#) == prefix + #"{"payload":{"mixed":["a"]},"points":[7]}"#) } - @Test("A duplicated row's default marker asks for a new id instead of failing as one") + @Test("A duplicated row's defaulted id asks for a new id instead of failing as one") func duplicatedRowGetsNewID() throws { let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ - QdrantCell(column: "id", text: "__DEFAULT__"), QdrantCell(column: "city", text: "Hanoi") + QdrantCell(column: "id", text: nil), QdrantCell(column: "city", text: "Hanoi") ]) #expect(try writes([change]).first?.contains(#""id":"00000000-0000-0000-0000-000000000001""#) == true) } + @Test("The text __DEFAULT__ in a payload field is stored as that text") + func defaultLookingTextIsAValue() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: "7"), QdrantCell(column: "city", text: "__DEFAULT__") + ]) + #expect(try writes([change]).first?.contains(#""city":"__DEFAULT__""#) == true) + } + @Test("A server that would overwrite a taken id gets no grid insert") func insertNeedsConditionalUpsert() { let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift index 76bf57dbd6..46f6d66fc2 100644 --- a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift @@ -19,6 +19,12 @@ struct TeradataReplyDecodingTests { return bytes } + private static func execute(_ reply: [UInt8]) throws -> TeradataResultSet { + let transport = StoppableTransport(replies: [LanMessage(kind: .start, body: reply).encoded()]) + let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + return try TeradataConnection(config: config, transport: transport).execute("SELECT c FROM t") + } + @Test("A result description whose CHAR column has length -1 fails the reply instead of crashing") func negativeColumnLengthFailsTheReply() throws { let reply = Self.reply(charLength: [UInt8](repeating: 0xFF, count: 8), record: [0x00, 0x78]) @@ -26,13 +32,13 @@ struct TeradataReplyDecodingTests { let parcels = try Parcel.decodeAll(reply) let flavors = parcels.map(\.flavor) #expect(flavors == [ParcelFlavor.prepInfoX.rawValue, ParcelFlavor.record.rawValue, ParcelFlavor.endRequest.rawValue]) - #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try TeradataResultParser.parse(parcels) } + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try Self.execute(reply) } } @Test("A CHAR column of the declared length still decodes") func charColumnDecodes() throws { let reply = Self.reply(charLength: [0, 0, 0, 0, 0, 0, 0, 3], record: [0x00, 0x61, 0x62, 0x63]) - let result = try TeradataResultParser.parse(try Parcel.decodeAll(reply)) + let result = try Self.execute(reply) #expect(result.rows == [[TeradataValue.text("abc")]]) } diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift index 4f19e1b292..76202549d8 100644 --- a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift @@ -4,7 +4,7 @@ import Testing // Replies are read in order; a hook keyed by a reply's index runs just before that reply is read, which is where a // request is waiting on the server when Stop is pressed. -private final class StoppableTransport: TeradataTransport { +final class StoppableTransport: TeradataTransport { private var replies: [[UInt8]] private var stream: [UInt8] = [] private var repliesRead = 0 @@ -97,6 +97,9 @@ private final class SilentTransport: TeradataTransport, @unchecked Sendable { struct TeradataStopTests { private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + private static let ansiConfig = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: .ansi + ) private static let start = MessageKind.start.rawValue private static let cont = MessageKind.cont.rawValue private static let abort = MessageKind.abort.rawValue @@ -128,16 +131,36 @@ struct TeradataStopTests { return true } - @Test("Stop while the server works on a request aborts it on the session, which stays logged on") + private static func isRolledBack(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .rolledBackByStop = wire else { return false } + return true + } + + @Test("A Stop outside a session set to ANSI reads as a cancel") + func stopOutsideANSIModeIsACancel() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + guard let wire = error as? TeradataWireError, case .cancelled = wire else { + Issue.record("Expected cancelled, got \(String(describing: error))") + return + } + } + + @Test("Stop while the server works on a request aborts it on the session, which stays logged on and reports the rollback") func stopAbortsTheRequestAndKeepsTheSession() throws { let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) let transport = StoppableTransport(replies: [aborted, Self.finished]) - let connection = TeradataConnection(config: Self.config, transport: transport) + let connection = TeradataConnection(config: Self.ansiConfig, transport: transport) transport.beforeReply[0] = { connection.cancel() } let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } - #expect(Self.isCancelled(error)) + #expect(Self.isRolledBack(error)) #expect(transport.sentKinds == [Self.start, Self.abort]) #expect(transport.sent.last?.requestNumber == transport.sent.first?.requestNumber) #expect(transport.sent.last?.body == TeradataMessages.abortParcel().encoded()) diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift new file mode 100644 index 0000000000..96e3f43823 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift @@ -0,0 +1,287 @@ +@testable import TableProTeradataCore +import Testing + +struct TeradataStreamTests { + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + private static let integer: UInt16 = 496 + private static let char: UInt16 = 452 + + private static let numbers = prepInfoX([(integer, 4, "n")]) + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func u16(_ value: Int) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + // A PrepInfoX parcel laid out as in TeradataReplyDecodingTests. + private static func prepInfoX(_ columns: [(type: UInt16, length: Int, name: String)]) -> Parcel { + var body = [UInt8](repeating: 0, count: 8) + u16(0) + u16(columns.count) + for column in columns { + let name = Array(column.name.utf8) + body += u16(Int(column.type)) + body += (0..<8).map { UInt8(truncatingIfNeeded: column.length >> (8 * (7 - $0))) } + body += [0, 0] + body += u16(name.count) + name + body += u16(0) + u16(0) + } + return Parcel(.prepInfoX, body: body) + } + + private static func number(_ value: Int32) -> Parcel { + let bits = UInt32(bitPattern: value) + return Parcel(.record, body: [0x00] + (0..<4).map { UInt8(truncatingIfNeeded: bits >> (8 * (3 - $0))) }) + } + + private static func code(_ text: String) -> Parcel { + Parcel(.record, body: [0x00] + Array(text.utf8)) + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + return Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + u16(code) + u16(text.count) + text) + } + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + private static func connection( + _ replies: [[UInt8]], + transactionMode: TeradataTransactionMode = .default + ) -> (TeradataConnection, StoppableTransport) { + let transport = StoppableTransport(replies: replies) + let config = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: transactionMode + ) + return (TeradataConnection(config: config, transport: transport), transport) + } + + @Test("Each reply's rows reach the consumer before the next Continue goes out") + func rowsArriveReplyByReply() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3)]), + Self.reply([Self.number(4), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches: [[[TeradataValue]]] = [] + var sentBeforeEachBatch: [Int] = [] + + try connection.stream("SELECT n FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + batches.append(rows) + sentBeforeEachBatch.append(transport.sent.count) + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == [[[.integer(1)], [.integer(2)]], [[.integer(3)]], [[.integer(4)]]]) + #expect(sentBeforeEachBatch == [1, 2, 3]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.dropFirst().allSatisfy { $0.body == TeradataMessages.responseParcel().encoded() }) + } + + @Test("A SELECT that matches no row still describes its columns") + func emptyResultStillHasColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Self.numbers, Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders WHERE 1 = 0", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in + batches += 1 + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == 0) + } + + @Test("A statement without a result set still answers, with no columns") + func statementWithoutResultHasNoColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + + try connection.stream("DELETE FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in true }) + + #expect(headers == [[]]) + } + + @Test("A consumer that wants no more rows ends the output and keeps the session") + func consumerEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + return false + }) + + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A failure after some rows throws the server's error and keeps the session") + func failureAfterRowsThrows() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Self.failure(code: 2_646, "No more spool space in dbc.")]), + Self.finished, + ]) + var batches: [[[TeradataValue]]] = [] + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { rows in + batches.append(rows) + return true + }) + } + + var code: Int? + if case .server(let failed, _)? = error as? TeradataWireError { code = failed } + #expect(code == 2_646) + #expect(batches == [[[.integer(1)]]]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("Stop between replies ends the output, and the stream ends as cancelled") + func stopBetweenRepliesEndsTheStream() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + connection.cancel() + return true + }) + } + + #expect(Self.isCancelled(error)) + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A later statement's records are never read against the first statement's columns") + func laterStatementKeepsItsOwnColumns() throws { + let answer = Self.reply([ + Self.numbers, Self.number(1), Parcel(.endStatement), + Self.prepInfoX([(Self.char, 3, "code")]), Self.code("abc"), Parcel(.endStatement), + Self.numbers, Self.number(2), Parcel(.endStatement), + Parcel(.endRequest), + ]) + let sql = "SELECT n FROM sales.a; SELECT code FROM sales.b; SELECT n FROM sales.c" + + let result = try Self.connection([answer]).0.execute(sql) + let names = result.columns.map { $0.name } + #expect(names == ["n"]) + #expect(result.rows == [[.integer(1)], [.integer(2)]]) + + var headers: [[String]] = [] + var streamed: [[TeradataValue]] = [] + try Self.connection([answer]).0.stream(sql, onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + streamed += rows + return true + }) + #expect(headers == [["n"]]) + #expect(streamed == result.rows) + } + + @Test("Stop before the first reply aborts an ANSI stream's request, and the stream reports the rollback") + func abortedStreamReportsTheRollback() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]), + Self.finished, + ], transactionMode: .ansi) + transport.beforeReply[0] = { connection.cancel() } + var headers = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in headers += 1 }, onRows: { _ in true }) + } + + var rolledBack = false + if case .rolledBackByStop? = error as? TeradataWireError { rolledBack = true } + #expect(rolledBack) + #expect(headers == 0) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + } + + @Test("A row cap ends the output at the reply that passes it and marks the result cut") + func rowCapEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3), Self.number(4)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 3) + + #expect(capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)], [.integer(3)]]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("A result with exactly as many rows as the cap is not cut") + func resultAtTheCapIsComplete() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2), Parcel(.endRequest)]), + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 2) + + #expect(!capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)]]) + #expect(transport.sentKinds == [Self.start]) + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift index c40818145b..eb1c3c1459 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift @@ -11,9 +11,6 @@ import OSLog public enum OracleCellFormatting { public static let maxHexBytes = 4_096 - /// A region-named zone is sent as an id into Oracle's time zone file, which the client does not have. - public static let timeZoneRegionPlaceholder = unsupportedPlaceholder(typeName: "time zone region") - // MARK: - Datetimes /// A zone's hour and minute are each sent above a bias, so a zone west of UTC sends bytes below it. @@ -32,16 +29,25 @@ public enum OracleCellFormatting { return fields.text(fractionDigits: fractionDigits) } - /// `TIMESTAMP(p) WITH TIME ZONE` as the wall clock at the value's own offset, followed by that offset - /// with no space. The date and time bytes are UTC. - public static func timestampWithTimeZoneText(wire bytes: [UInt8], fractionDigits: Int) -> String? { + // The date and time bytes are UTC, whether the zone is an offset or a region. + public static func timestampWithTimeZoneText( + wire bytes: [UInt8], + fractionDigits: Int, + regions: OracleTimeZoneRegions = .none + ) -> String? { guard var fields = OracleWireDateTime(bytes) else { return nil } + if let id = OracleTimeZoneRegions.regionID(wire: bytes) { + guard let region = regions.region(forID: id), let zone = region.zone else { + return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: 0) + } + let instant = Date(timeIntervalSince1970: TimeInterval(fields.secondsTreatingFieldsAsUTC)) + fields.shift(bySeconds: zone.secondsFromGMT(for: instant)) + // A space before the name, as Oracle's TZR spells it. + return fields.text(fractionDigits: fractionDigits) + " " + region.name + } guard bytes.count >= 13, bytes[11] != 0, bytes[12] != 0 else { return fields.text(fractionDigits: fractionDigits) + "+00:00" } - if bytes[11] & 0x80 != 0 { - return timeZoneRegionPlaceholder - } let offsetMinutes = (Int(bytes[11]) - zoneHourBias) * 60 + Int(bytes[12]) - zoneMinuteBias fields.shift(bySeconds: offsetMinutes * 60) return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: offsetMinutes) @@ -324,11 +330,12 @@ struct OracleResultColumn: Sendable, Equatable { enum OracleCellDecoding { private static let logger = Logger(subsystem: "com.TablePro", category: "OracleCellDecoding") - /// `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. + // `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. static func decode( _ cell: OracleCell, column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions = .none, unsupported: (String) -> Void ) -> OracleRawCell { guard let bytes = cell.bytes else { return .null } @@ -339,7 +346,7 @@ enum OracleCellDecoding { do { guard let text = try text(of: cell, wire: Array(bytes.readableBytesView), column: column, zones: zones, - unsupported: unsupported) + regions: regions, unsupported: unsupported) else { return .null } return .string(text) } catch { @@ -355,6 +362,7 @@ enum OracleCellDecoding { wire: [UInt8], column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, unsupported: (String) -> Void ) throws -> String? { switch cell.dataType { @@ -380,7 +388,9 @@ enum OracleCellDecoding { return OracleCellFormatting.timestampText(wire: wire, fractionDigits: column.fractionDigits) case .timestampTZ: - return OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: column.fractionDigits) + return OracleCellFormatting.timestampWithTimeZoneText( + wire: wire, fractionDigits: column.fractionDigits, regions: regions + ) case .timestampLTZ: return OracleCellFormatting.localTimestampText(wire: wire, fractionDigits: column.fractionDigits, zones: zones) diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift index b7b8433292..15758e58f6 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift @@ -109,9 +109,10 @@ public final class OracleCoreConnection: @unchecked Sendable { var close = OracleCloseRecord() var serverRelease: OracleServerRelease? var loginWarning: String? - /// Read before the first statement of each session, and again before the statement after one that can - /// change them, so no result ever waits on them mid-stream. + // Read before a session's first statement and again after one that can change them, never mid-stream. var sessionZones: OracleSessionTimeZones? + // The server's time zone file fixes these, so only a new session clears them. + var timeZoneRegions: OracleTimeZoneRegions? var isReservedForDebugging = false } @@ -218,6 +219,7 @@ public final class OracleCoreConnection: @unchecked Sendable { current.serverRelease = release current.loginWarning = connection.loginWarning?.message current.sessionZones = nil + current.timeZoneRegions = nil current.close.clearOnConnect() return true } @@ -846,8 +848,13 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) let result = try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) + let regions = await timeZoneRegions(on: connection) return try await collectRows( - statement, options: OracleRowLimits.collecting(admitted.options), zones: zones, on: connection + statement, + options: OracleRowLimits.collecting(admitted.options), + zones: zones, + regions: regions, + on: connection ) } await recordSuccess(of: role, on: admitted.session) @@ -936,7 +943,7 @@ public final class OracleCoreConnection: @unchecked Sendable { } } - /// The app's own session statements, none of which reads a `TIMESTAMP WITH LOCAL TIME ZONE`. + // The app's own session statements, none of which reads a zoned timestamp. private func collectRows( _ query: String, options: StatementOptions, @@ -946,6 +953,7 @@ public final class OracleCoreConnection: @unchecked Sendable { OracleStatement(unsafeSQL: query), options: options, zones: .driverDefault, + regions: OracleTimeZoneRegions.none, on: connection ) } @@ -954,12 +962,13 @@ public final class OracleCoreConnection: @unchecked Sendable { _ statement: OracleStatement, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, on connection: OracleNIO.OracleConnection ) async throws -> OracleRawResult { let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) var allRows: [[OracleRawCell]] = [] var truncated = false @@ -1005,10 +1014,12 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) + let regions = await timeZoneRegions(on: connection) try await streamRows( query, options: admitted.options, zones: zones, + regions: regions, batching: batching, on: connection, continuation: continuation @@ -1031,6 +1042,7 @@ public final class OracleCoreConnection: @unchecked Sendable { _ query: String, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, batching: OracleRowBatcher, on connection: OracleNIO.OracleConnection, continuation: AsyncThrowingStream.Continuation @@ -1039,7 +1051,7 @@ public final class OracleCoreConnection: @unchecked Sendable { let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) continuation.yield(.header(columns: columns.map(\.descriptor))) var batcher = batching @@ -1104,17 +1116,68 @@ public final class OracleCoreConnection: @unchecked Sendable { state.withLock { $0.sessionZones = nil } } + // MARK: - Time Zone Regions + + // A failed read keeps an empty map for the session, so a region value reads as its UTC wall clock and the + // read is not retried before every statement. + private func timeZoneRegions(on connection: OracleNIO.OracleConnection) async -> OracleTimeZoneRegions { + if let regions = state.withLock({ $0.timeZoneRegions }) { + return regions + } + do { + let regions = try await readTimeZoneRegions(on: connection) + state.withLock { $0.timeZoneRegions = regions } + return regions + } catch is CancellationError { + return OracleTimeZoneRegions.none + } catch { + osLogger.error( + "Reading the Oracle time zone regions failed: \(String(describing: type(of: error)), privacy: .public)" + ) + state.withLock { $0.timeZoneRegions = OracleTimeZoneRegions.none } + return OracleTimeZoneRegions.none + } + } + + // Read straight from the stream: the cell decode would turn the region column back into the UTC fallback. + private func readTimeZoneRegions(on connection: OracleNIO.OracleConnection) async throws -> OracleTimeZoneRegions { + let stream = try await connection.execute( + OracleStatement(unsafeSQL: OracleTimeZoneRegions.query), + options: OracleRowLimits.collecting(Self.sessionSetupOptions), + logger: nioLogger + ) + var rows: [(name: String, wire: [UInt8])] = [] + for try await row in stream { + let cells = Array(row) + guard cells.count == 2, let name = try? cells[0].decode(String.self), let wire = cells[1].bytes else { + continue + } + rows.append((name: name, wire: Array(wire.readableBytesView))) + } + let regions = OracleTimeZoneRegions(rows: rows) + let repeated = regions.duplicateIDs + let unknown = regions.unknownNames + if repeated > 0 || unknown > 0 { + osLogger.notice( + "Oracle time zone regions: \(regions.count, privacy: .public), repeated ids \(repeated, privacy: .public), unknown names \(unknown, privacy: .public)" + ) + } + return regions + } + // MARK: - Cell Decoding - /// Decodes the cells of one result, collecting the names of the types it had to render as placeholders. + // Collects the names of the types it had to render as placeholders, so each is logged once. private struct CellDecoder { let columns: [OracleResultColumn] let zones: OracleSessionTimeZones + let regions: OracleTimeZoneRegions private(set) var unsupportedTypeNames: Set = [] - init(columns: [OracleResultColumn], zones: OracleSessionTimeZones) { + init(columns: [OracleResultColumn], zones: OracleSessionTimeZones, regions: OracleTimeZoneRegions) { self.columns = columns self.zones = zones + self.regions = regions } mutating func decode(_ cells: some Sequence) -> [OracleRawCell] { @@ -1124,7 +1187,7 @@ public final class OracleCoreConnection: @unchecked Sendable { let column = columns.indices.contains(index) ? columns[index] : OracleResultColumn(name: cell.columnName, dataType: cell.dataType, scale: 9) - values.append(OracleCellDecoding.decode(cell, column: column, zones: zones) { name in + values.append(OracleCellDecoding.decode(cell, column: column, zones: zones, regions: regions) { name in unsupportedTypeNames.insert(name) }) } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift new file mode 100644 index 0000000000..34ec98233f --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift @@ -0,0 +1,63 @@ +import Foundation + +// A TIMESTAMP WITH TIME ZONE stored with a region name carries an id into the server's time zone file, which a +// thin client does not have, so the names are read from the server once per connection. +public struct OracleTimeZoneRegions: Sendable, Equatable { + public struct Region: Sendable, Equatable { + public let name: String + public let zone: TimeZone? + } + + public static let none = OracleTimeZoneRegions(regions: [:], duplicateIDs: 0) + + // AT TIME ZONE converts an instant, so no name can fail on a wall clock its zone skips. + public static let query = "SELECT TZNAME, TIMESTAMP '2000-01-01 00:00:00 UTC' AT TIME ZONE TZNAME FROM " + + "(SELECT DISTINCT TZNAME FROM \(OracleDictionary.performanceView("V$TIMEZONE_NAMES"))) ORDER BY TZNAME" + + private static let regionFlag: UInt8 = 0x80 + + let regions: [Int: Region] + let duplicateIDs: Int + + private init(regions: [Int: Region], duplicateIDs: Int) { + self.regions = regions + self.duplicateIDs = duplicateIDs + } + + public init(rows: [(name: String, wire: [UInt8])]) { + var regions: [Int: Region] = [:] + var duplicateIDs = 0 + for row in rows { + guard let id = Self.regionID(wire: row.wire) else { continue } + guard regions[id] == nil else { + duplicateIDs += 1 + continue + } + regions[id] = Region(name: row.name, zone: TimeZone(identifier: row.name)) + } + self.init(regions: regions, duplicateIDs: duplicateIDs) + } + + public var isEmpty: Bool { + regions.isEmpty + } + + public var count: Int { + regions.count + } + + var unknownNames: Int { + regions.values.filter { $0.zone == nil }.count + } + + func region(forID id: Int) -> Region? { + regions[id] + } + + // An offset's hour byte is at most 20 + 14, so the flag alone marks a region; the low two bits of byte 12 are not + // part of the id. + public static func regionID(wire bytes: [UInt8]) -> Int? { + guard bytes.count >= 13, bytes[11] & regionFlag != 0 else { return nil } + return (Int(bytes[11] & 0x7F) << 6) | (Int(bytes[12] & 0xFC) >> 2) + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift index dbc3ed24d6..5e5f2c8306 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift @@ -15,12 +15,15 @@ struct OracleCellDecodingTests { _ bytes: [UInt8]?, as type: OracleDataType, declared: OracleDataType? = nil, - scale: Int = 0 + scale: Int = 0, + regions: OracleTimeZoneRegions = .none ) -> (OracleRawCell, Set) { let cell = OracleCell(bytes: bytes.map { ByteBuffer(bytes: $0) }, dataType: type, columnName: "C", columnIndex: 0) let column = OracleResultColumn(name: "C", dataType: declared ?? type, scale: scale) var unsupported: Set = [] - let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven) { unsupported.insert($0) } + let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven, regions: regions) { + unsupported.insert($0) + } return (value, unsupported) } @@ -73,6 +76,14 @@ struct OracleCellDecodingTests { #expect(decode(Self.hex("787c0102020506000003e8"), as: .timestampLTZ, scale: 6).0 == .string("2024-01-02 08:04:05.000001+07:00")) } + @Test func testRegionZoneDecodesThroughTheRegionMap() { + let paris = Self.hex("787c01020305061dcd650085f8") + let (named, unsupported) = decode(paris, as: .timestampTZ, scale: 6, regions: OracleTimeZoneRegionFixtures.paris) + #expect(named == .string("2024-01-02 03:04:05.500000 Europe/Paris")) + #expect(unsupported.isEmpty) + #expect(decode(paris, as: .timestampTZ, scale: 6).0 == .string("2024-01-02 02:04:05.500000+00:00")) + } + @Test func testUniversalRowIDReadsAsOracleSpellsIt() { #expect(decode(Array("*BAYAAesCwQL+".utf8), as: .uRowID).0 == .string("*BAYAAesCwQL+")) } diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift index 96687599e5..a766acd508 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift @@ -54,10 +54,58 @@ struct OracleCellFormattingTests { #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: utc, fractionDigits: 0) == "2024-01-01 00:30:00+05:00") } - /// A zone stored by region name is an id into Oracle's time zone file. It used to read ``. - @Test func testRegionNamedZoneIsAPlaceholder() { - let paris: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] - #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: paris, fractionDigits: 6) == "") + private static let parisInJanuary: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] + + private static func regionText(_ wire: [UInt8], fractionDigits: Int = 6, regions: OracleTimeZoneRegions) -> String? { + OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: fractionDigits, regions: regions) + } + + // A zone stored by region name used to read ``, and export wrote that text. + @Test func testRegionZoneIsTheWallClockAndOraclesName() { + #expect(Self.regionText(Self.parisInJanuary, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + @Test func testRegionZoneFollowsDaylightSaving() { + let july: [UInt8] = [0x78, 0x7c, 0x07, 0x01, 0x0d, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + #expect(Self.regionText(july, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) == "2024-07-01 14:00:00 Europe/Paris") + } + + @Test func testRegionZoneCanCrossAYearBoundary() { + let lastHalfHourOf2024: [UInt8] = [0x78, 0x7c, 0x0c, 0x1f, 0x18, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + let paris = Self.regionText(lastHalfHourOf2024, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) + #expect(paris == "2025-01-01 00:30:00 Europe/Paris") + + // Id 65 is made up; only the map decides which name it stands for. + let losAngeles = OracleTimeZoneRegions(rows: [OracleTimeZoneRegionFixtures.row("America/Los_Angeles", zone: (0x81, 0x04))]) + let threeAMOn2025: [UInt8] = [0x78, 0x7d, 0x01, 0x01, 0x04, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x81, 0x04] + #expect(Self.regionText(threeAMOn2025, fractionDigits: 0, regions: losAngeles) == "2024-12-31 19:00:00 America/Los_Angeles") + } + + @Test func testRegionZoneIgnoresTheLowBitsOfByte12() { + var lowBitsSet = Self.parisInJanuary + lowBitsSet[12] = 0xf9 + #expect(Self.regionText(lowBitsSet, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + // Without a name the text keeps the instant, as the UTC wall clock in the offset spelling the grid reads. + @Test func testRegionMissingFromTheMapReadsAsUTC() { + var otherID = Self.parisInJanuary + otherID[12] = 0xfc + #expect(Self.regionText(otherID, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 02:04:05.500000+00:00") + #expect(Self.regionText(Self.parisInJanuary, regions: .none) == "2024-01-02 02:04:05.500000+00:00") + #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: Self.parisInJanuary, fractionDigits: 6) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionFoundationDoesNotKnowReadsAsUTC() { + let unknown = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + #expect(Self.regionText(Self.parisInJanuary, regions: unknown) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionsLeaveAnOffsetValueAlone() { + let minusFiveThirty: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x09, 0x23, 0x06, 0x07, 0x5b, 0xca, 0x00, 0x0f, 0x1e] + #expect(Self.regionText(minusFiveThirty, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.123456-05:30") } /// The bytes are the value in the database's zone; the text is the session's wall clock and the session's @@ -113,6 +161,7 @@ struct OracleCellFormattingTests { "2024-01-02 03:04:05.123456-05:30", "2024-01-02 08:04:05.000001+07:00", "2024-07-01 14:00:00+02:00", + "2024-01-02 02:04:05.500000+00:00", ] for spelling in spellings { let range = NSRange(spelling.startIndex..., in: spelling) diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift new file mode 100644 index 0000000000..da3f079302 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift @@ -0,0 +1,77 @@ +import Foundation +@testable import TableProOracleCore +import Testing + +enum OracleTimeZoneRegionFixtures { + // Bytes 11 and 12 of a Europe/Paris value measured on 23ai: region id 382. + static let parisZone: (UInt8, UInt8) = (0x85, 0xf8) + + // A row as the map query returns it: 2000-01-01 00:00:00 UTC at the named zone. + static func row(_ name: String, zone: (UInt8, UInt8)) -> (name: String, wire: [UInt8]) { + (name, [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, zone.0, zone.1]) + } + + static let paris = OracleTimeZoneRegions(rows: [row("Europe/Paris", zone: parisZone)]) +} + +struct OracleTimeZoneRegionsTests { + private static func wire(zone: (UInt8, UInt8)) -> [UInt8] { + OracleTimeZoneRegionFixtures.row("", zone: zone).wire + } + + @Test func testRegionIDSpansBytes11And12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: OracleTimeZoneRegionFixtures.parisZone)) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0xff, 0xfc))) == 8_191) + } + + @Test func testRegionIDIgnoresTheLowBitsOfByte12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xf9))) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xfb))) == 382) + } + + // The flag alone marks a region, so an id whose low six bits are zero is not read as a zero offset. + @Test func testRegionIDWithAZeroByte12IsStillARegion() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0x00))) == 320) + } + + @Test func testOffsetZonesHaveNoRegionID() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x14, 0x3c))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x0f, 0x1e))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x00, 0x00))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Array(Self.wire(zone: (0x85, 0xf8)).prefix(11))) == nil) + } + + @Test func testMapDropsOffsetRowsAndKeepsTheFirstOfARepeatedID() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Europe/Paris", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("Europe/Monaco", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("UTC", zone: (0x14, 0x3c)), + ]) + #expect(regions.count == 1) + #expect(regions.duplicateIDs == 1) + let paris = try #require(regions.region(forID: 382)) + #expect(paris.name == "Europe/Paris") + #expect(paris.zone?.identifier == "Europe/Paris") + } + + @Test func testMapKeepsANameFoundationDoesNotKnowWithoutAZone() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + let region = try #require(regions.region(forID: 382)) + #expect(region.name == "Not/AZone") + #expect(region.zone == nil) + #expect(regions.unknownNames == 1) + } + + @Test func testNoneHoldsNoRegions() { + #expect(OracleTimeZoneRegions.none.isEmpty) + #expect(OracleTimeZoneRegions(rows: []) == OracleTimeZoneRegions.none) + } + + // A schema can hold its own V$TIMEZONE_NAMES, which an unqualified name resolves to first. + @Test func testQueryReadsTheQualifiedPerformanceView() { + #expect(OracleTimeZoneRegions.query.contains("SELECT DISTINCT TZNAME FROM SYS.V_$TIMEZONE_NAMES)")) + #expect(OracleTimeZoneRegions.query.hasSuffix("ORDER BY TZNAME")) + } +} diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift index 5bbd4314cd..103e191ecc 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift @@ -71,6 +71,7 @@ internal final class BigQueryPluginDriver: PluginDatabaseDriver, @unchecked Send .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift index 0ee163ee61..c75a935967 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift @@ -6,8 +6,6 @@ import TableProPluginKit internal struct BigQueryStatementGenerator { typealias Statement = (statement: String, parameters: [PluginCellValue]) - static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "BigQueryStatementGenerator") private static let placeholder = "?" @@ -81,7 +79,7 @@ internal struct BigQueryStatementGenerator { var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, column) in columns.enumerated() { - guard let value = values[column], !Self.isDefaultSentinel(value) else { continue } + guard let value = values[column], !change.isColumnDefault(index) else { continue } names.append(GoogleSQLLiteral.quotedIdentifier(column)) if let sql = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(sql) @@ -124,7 +122,7 @@ internal struct BigQueryStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = GoogleSQLLiteral.quotedIdentifier(cellChange.columnName) - guard !Self.isDefaultSentinel(cellChange.newValue) else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) @@ -185,10 +183,6 @@ internal struct BigQueryStatementGenerator { ) } - private static func isDefaultSentinel(_ value: PluginCellValue) -> Bool { - value.asText == defaultSentinel - } - // The host checked a marked cell's text against the engine's vocabulary before handing it over. private static func expression(_ value: PluginCellValue, isMarked: Bool) -> String? { guard isMarked, case .text(let sql) = value else { return nil } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift index 6f92b8c0a4..78becc1c1e 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -3,8 +3,6 @@ import TableProBigtableCore import TableProPluginKit extension BigtablePluginDriver { - private static let defaultMarker = "__DEFAULT__" - func generateRowWrites( table: String, schema: String?, @@ -50,11 +48,13 @@ extension BigtablePluginDriver { switch change.type { case .insert: return try BigtableRowWrites.insert( - table: table, columns: columns, values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent) + table: table, + columns: columns, + values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent.union(change.defaultColumns ?? [])) ) case .update: let changes = try change.cellChanges.map { cell in - guard cell.newValue.asText != Self.defaultMarker else { + guard !change.isColumnDefault(cell.columnIndex) else { throw BigtableError.unsupported(String(localized: "Bigtable has no default values. Delete the cell, or enter a value.")) } return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) @@ -74,11 +74,10 @@ extension BigtablePluginDriver { return key } - /// A new row holds the host's Default marker where it has no value yet. Bigtable generates - /// nothing, so that cell is left out, and a `_key` left that way is refused. + // Bigtable generates nothing, so a cell left to the default is left out, and a `_key` left that way is refused. private func cells(_ row: [PluginCellValue], absent: Set) -> [BigtableRowSet.Cell] { row.enumerated().map { index, value in - absent.contains(index) || value.asText == Self.defaultMarker ? .absent : Self.cell(value) + absent.contains(index) ? .absent : Self.cell(value) } } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift index 6a6b3de9c0..c67626f445 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -22,7 +22,7 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send } var serverVersion: String? { "Google Cloud Bigtable" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index ff8717faec..64b527cbf6 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -172,6 +172,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen .alterTableDDL, .cellExpressions, .uuidCellText, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index ca4edf9125..0f9731a695 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -11,7 +11,6 @@ import TableProPluginKit // A new row is written `IF NOT EXISTS` and an edit `IF EXISTS` (CassandraConditionalWrite), and anything CQL cannot // do to a row is refused by name rather than sent. enum CassandraRowWriter { - static let defaultSentinel = "__DEFAULT__" static let unappliedMarker = CassandraConditionalWrite.appliedColumn static func rowWrites( @@ -79,12 +78,16 @@ enum CassandraRowWriter { )) } let row = values ?? rowFromCellChanges(change) + // A new row built from its edits leaves every column it never set to the server. + let leftOut = (change.defaultColumns ?? []).union( + values == nil ? Set(columns.indices).subtracting(change.cellChanges.map(\.columnIndex)) : [] + ) let expressionColumns = change.expressionColumns ?? [] var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in row.enumerated() where index < columns.count { - if value.asText == defaultSentinel { continue } + if leftOut.contains(index) { continue } names.append(CassandraBrowseRenderer.quote(columns[index])) if let call = expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(call) @@ -95,7 +98,7 @@ enum CassandraRowWriter { } for keyColumn in primaryKeyColumns { guard let index = columns.firstIndex(of: keyColumn), index < row.count, !row[index].isNull, - row[index].asText != defaultSentinel + !leftOut.contains(index) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "Enter a value for the primary key column %@."), keyColumn @@ -120,7 +123,7 @@ enum CassandraRowWriter { cell.columnName )) } - if cell.newValue.asText == defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( localized: "Cassandra columns have no default value." )) @@ -168,7 +171,7 @@ enum CassandraRowWriter { } private func rowFromCellChanges(_ change: PluginRowChange) -> [PluginCellValue] { - var row = [PluginCellValue](repeating: .text(defaultSentinel), count: columns.count) + var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift index b0ea856105..5166531fd1 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift @@ -223,6 +223,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked .uuidCellText, .repeatingPrimaryKey, .readOnlyStatements, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift index 94d7513dfc..961f6f6185 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift @@ -17,8 +17,6 @@ nonisolated enum ClickHouseRowWrites { let delete: (_ table: String, _ predicate: String) -> String } - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func writes( table: String, schema: String?, @@ -52,7 +50,8 @@ nonisolated enum ClickHouseRowWrites { let values = insertedRowData[change.rowIndex] else { continue } writes.append(insert( into: target, columns: columns, values: values, expressions: change.expressionColumns ?? [], - serverOwned: context.serverOwnedColumns, rowIndex: change.rowIndex + defaults: change.defaultColumns ?? [], serverOwned: context.serverOwnedColumns, + rowIndex: change.rowIndex )) case .update: guard !change.cellChanges.isEmpty else { continue } @@ -78,13 +77,14 @@ nonisolated enum ClickHouseRowWrites { columns: [String], values: [PluginCellValue], expressions: Set, + defaults: Set = [], serverOwned: Set = [], rowIndex: Int ) -> PluginRowWrite { var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] - for (index, value) in values.enumerated() where index < columns.count && value != defaultMarker { + for (index, value) in values.enumerated() where index < columns.count && !defaults.contains(index) { names.append(clickHouseQuotedIdentifier(columns[index])) if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift index 6c16be41f6..d1ef570299 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift @@ -248,6 +248,7 @@ struct ClickHouseDMLStatementTests { func insertStaysPlain() throws { var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) change.expressionColumns = [1] + change.defaultColumns = [2] let write = try #require(try writes( [change], columns: ["id", "at", "note"], inserted: [0: ["7", "now()", "__DEFAULT__"]] )?.first) @@ -257,9 +258,21 @@ struct ClickHouseDMLStatementTests { #expect(write.lookup == nil) } + @Test("The text __DEFAULT__ in a column with no default mark is written as text") + func unmarkedDefaultTextIsAValue() throws { + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let write = try #require(try writes( + [change], columns: ["id", "note"], inserted: [0: ["7", "__DEFAULT__"]] + )?.first) + + #expect(write.statement == "INSERT INTO `users` (`id`, `note`) VALUES (?, ?)") + #expect(write.parameters == ["7", "__DEFAULT__"]) + } + @Test("A new row that sets nothing takes every default, named through a column the server does not fill") func emptyInsertTakesDefaults() throws { - let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + change.defaultColumns = [0, 1] var context = PluginRowWriteContext() context.serverOwnedColumns = ["id"] let write = try #require(try waitingDriver.generateRowWrites( diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json index 89595dcb46..2c58ed400f 100644 --- a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json @@ -146,7 +146,8 @@ { "type": "TIMESTAMP WITH TIME ZONE", "values": [ - {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"} + {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"}, + {"sql": "TIMESTAMP '2026-01-01 00:00:00 Europe/Paris'", "cell": "2026-01-01 00:00:00.000000 Europe/Paris"} ], "edit": {"text": "2000-01-01 00:00:00.000000+00:00", "cell": "2000-01-01 00:00:00.000000+00:00"}, "known": { diff --git a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift index 70b89896ec..46a2450be2 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -160,7 +160,7 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var capabilities: PluginCapabilities { [ .parameterizedQueries, .transactions, .alterTableDDL, .multiSchema, .schemaCompare, .dataCompare, - .rowWrites, .cellExpressions, + .rowWrites, .cellExpressions, .columnDefaultMarks, ] } diff --git a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift index cb50bfa039..f06963b60f 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift @@ -19,7 +19,10 @@ extension DamengPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex), let row = insertedRowData[change.rowIndex], - let statement = insertStatement(table: qualifiedTable, columns: columns, values: row, expressions: expressions) + let statement = insertStatement( + table: qualifiedTable, columns: columns, values: row, expressions: expressions, + defaults: change.defaultColumns ?? [] + ) else { continue } writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: @@ -28,7 +31,7 @@ extension DamengPluginDriver { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cell -> String in let column = quoteIdentifier(cell.columnName) - if cell.newValue == Self.defaultMarker { return "\(column) = DEFAULT" } + if change.isColumnDefault(cell.columnIndex) { return "\(column) = DEFAULT" } if expressions.contains(cell.columnIndex), case .text(let sql) = cell.newValue { return "\(column) = \(sql)" } parameters.append(cell.newValue) return "\(column) = ?" @@ -61,11 +64,11 @@ extension DamengPluginDriver { rows: [[PluginCellValue]] ) -> [(statement: String, parameters: [PluginCellValue])]? { let qualifiedTable = qualifiedName(schema: schema, object: table) - return rows.compactMap { insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: []) } + return rows.compactMap { + insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: [], defaults: []) + } } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") - func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { guard !definition.columns.isEmpty else { return nil } let table = qualifiedName(schema: nil, object: definition.tableName) @@ -200,14 +203,15 @@ extension DamengPluginDriver { table: String, columns: [String], values: [PluginCellValue], - expressions: Set + expressions: Set, + defaults: Set ) -> (statement: String, parameters: [PluginCellValue])? { var insertColumns: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in values.enumerated() where index < columns.count { insertColumns.append(quoteIdentifier(columns[index])) - if value == Self.defaultMarker { + if defaults.contains(index) { placeholders.append("DEFAULT") } else if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift index 41e0c101de..e02600add0 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift @@ -34,7 +34,7 @@ final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return client.endpoint.isLocal ? "DynamoDB Local" : "DynamoDB \(client.endpoint.signingRegion)" } - var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery] } + var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift index 85756fc6c9..07585c2818 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift @@ -10,8 +10,6 @@ import TableProSQLQuoting /// the grid loaded, so it neither recreates an item someone deleted nor overwrites one someone /// changed. struct DynamoDBWriteStatements { - static let defaultSentinel = "__DEFAULT__" - let table: String let columns: [String] let keyColumns: [String] @@ -55,13 +53,12 @@ struct DynamoDBWriteStatements { return writes.map { (statement: $0.statement, parameters: $0.parameters) } } - func insert(row: [PluginCellValue]) -> (statement: String, parameters: [PluginCellValue]) { + func insert(row: [PluginCellValue], defaults: Set = []) -> (statement: String, parameters: [PluginCellValue]) { var names: [String] = [] var parameters: [PluginCellValue] = [] - for (index, column) in columns.enumerated() where index < row.count { + for (index, column) in columns.enumerated() where index < row.count && !defaults.contains(index) { let value = row[index] if case .null = value { continue } - if case .text(let text) = value, text == Self.defaultSentinel { continue } names.append(column) parameters.append(value) } @@ -73,14 +70,15 @@ struct DynamoDBWriteStatements { _ change: PluginRowChange, rowData: [PluginCellValue]? ) -> (statement: String, parameters: [PluginCellValue]) { + let defaults = change.defaultColumns ?? [] if let rowData { - return insert(row: rowData) + return insert(row: rowData, defaults: defaults) } var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } - return insert(row: row) + return insert(row: row, defaults: defaults) } private func update(_ change: PluginRowChange) throws -> (statement: String, parameters: [PluginCellValue])? { @@ -92,7 +90,7 @@ struct DynamoDBWriteStatements { var guardValues: [PluginCellValue] = [] for cell in change.cellChanges { - if case .text(let text) = cell.newValue, text == Self.defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "DynamoDB has no default values, so %@ cannot be set to DEFAULT."), cell.columnName @@ -239,7 +237,7 @@ struct DynamoDBParameterBinder { } return .binary(data) case .text(let text): - guard !text.isEmpty, text != DynamoDBWriteStatements.defaultSentinel else { + guard !text.isEmpty else { throw DynamoDBError.invalidValue( attribute: attribute, reason: String(localized: "A key attribute needs a value") ) diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift index 2d51700557..bb48629513 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift @@ -35,7 +35,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -248,6 +248,11 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes.filter { $0.type != .insert }) { + throw refusal + } + var insertedRowData = insertedRowData + let changes = changes.map { Self.leavingDefaultsOut($0, insertedRowData: &insertedRowData) } let typeNames = columnTypeNames(for: columns, index: table) let generator = ElasticsearchStatementGenerator( index: table, @@ -263,6 +268,29 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked ) } + // A new document leaves out a field left to its default, and one whose `_id` is left to it gets an id from the server. + private static func leavingDefaultsOut( + _ change: PluginRowChange, + insertedRowData: inout [Int: [PluginCellValue]] + ) -> PluginRowChange { + guard change.type == .insert, let defaults = change.defaultColumns, !defaults.isEmpty else { return change } + if var row = insertedRowData[change.rowIndex] { + for index in defaults where row.indices.contains(index) { row[index] = .null } + insertedRowData[change.rowIndex] = row + } + var cleared = PluginRowChange( + rowIndex: change.rowIndex, + type: change.type, + cellChanges: change.cellChanges.map { cell in + defaults.contains(cell.columnIndex) ? (cell.columnIndex, cell.columnName, cell.oldValue, .null) : cell + }, + originalRow: change.originalRow + ) + cleared.absentColumns = change.absentColumns + cleared.expressionColumns = change.expressionColumns + return cleared + } + // MARK: - Caching func cachedMappingColumns(_ index: String) async throws -> [ElasticsearchColumn] { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index 038ed7dbcc..9544d1e405 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -51,7 +51,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { false } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } // etcd has no transactions. @@ -449,6 +449,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let generator = EtcdStatementGenerator( prefix: keyspace.prefix(forTable: table), columns: columns diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift index 3e89356275..0dfd9c9ccc 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift @@ -35,6 +35,7 @@ extension FirebasePluginDriver { insertedRowIndices: Set, context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let types = writeColumnTypes(table, host: context.columnTypeNames) return try changes.compactMap { change in do { diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift index abad5c8f93..daa0134f66 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift @@ -34,7 +34,7 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send variant == .firestore ? "Cloud Firestore" : "Firebase Realtime Database" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift index 962b4b563c..c1a4784456 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift @@ -121,6 +121,42 @@ struct HanaExplainRequest: Encodable, Sendable { let timeoutSeconds: Int } +struct HanaCursorRequest: Encodable, Sendable { + let sql: String + let timeoutSeconds: Int + let batchRows: Int +} + +struct HanaCursorHeader: Decodable, Equatable, Sendable { + let columns: [String] + let columnTypeNames: [String] + let columnClassifications: [String?] + let rowsAffected: Int64 + let hasResultSet: Bool +} + +struct HanaCursorBatch: Decodable, Equatable, Sendable { + let rows: [[HanaBridgeCell]] + let truncatedLobCount: Int + let done: Bool + let sessionLost: Bool +} + +struct HanaCursorOpening: Decodable, Sendable { + let header: HanaCursorHeader + let batch: HanaCursorBatch + + // The helper sends the header and the first batch as one flat object. + init(from decoder: any Decoder) throws { + header = try HanaCursorHeader(from: decoder) + batch = try HanaCursorBatch(from: decoder) + } +} + +struct HanaCursorClosing: Decodable, Sendable { + let sessionLost: Bool +} + struct HanaConnectResult: Decodable, Equatable, Sendable { let serverVersion: String let currentSchema: String diff --git a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift index f129f13637..e8cfcba775 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift @@ -21,6 +21,7 @@ final class HanaConnection: HanaSession, @unchecked Sendable { var hasLostConnection = false } + private static let cursorBatchRows = 1_000 private static let logger = Logger(subsystem: "com.TablePro", category: "HanaConnection") private let bridge: any HanaNativeBridge @@ -105,6 +106,28 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws { + let request = HanaCursorRequest(sql: sql, timeoutSeconds: queryTimeoutSeconds, batchRows: Self.cursorBatchRows) + let requestJSON = try JSONEncoder().encode(request) + // Every fetch stays in this one queue item. The cursor holds the helper's operation slot, so a read queued + // between two fetches would wait on that slot while the next fetch waited behind it on this queue. + try await perform(.connected, cancellation: cancellation) { bridge, ticket in + try self.drainCursor( + bridge, + ticket: ticket, + request: requestJSON, + cancellation: cancellation, + onHeader: onHeader, + onBatch: onBatch + ) + } + } + func cancel(_ cancellation: HanaOperationSlot) { guard let ticket = cancellation.cancel() else { return } bridge.cancel(ticket) @@ -195,6 +218,65 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + private func drainCursor( + _ bridge: any HanaNativeBridge, + ticket: HanaOperationTicket, + request: Data, + cancellation: HanaOperationSlot, + onHeader: (HanaCursorHeader) -> Void, + onBatch: (HanaCursorBatch) -> Void + ) throws { + let opened = try bridge.openCursor(ticket, request: request) + // A failure reply means the helper already closed the cursor. Any other exit leaves it open until closed here. + var holdsCursor = true + defer { + if holdsCursor { + closeCursor(bridge, ticket: ticket) + } + } + let opening = try Self.decode(HanaCursorOpening.self, from: opened) + onHeader(opening.header) + var batch = opening.batch + while true { + if batch.sessionLost { + Self.logger.error("SAP HANA session was lost while its cursor was open") + markLost(ticket.session) + } + if !batch.rows.isEmpty { + onBatch(batch) + } + if batch.done { + holdsCursor = false + return + } + guard !cancellation.isCancelled else { + throw HanaBridgeFailure(kind: .cancelled) + } + let fetched: Data + do { + fetched = try bridge.fetch(ticket) + } catch { + holdsCursor = false + throw error + } + batch = try Self.decode(HanaCursorBatch.self, from: fetched) + } + } + + private func closeCursor(_ bridge: any HanaNativeBridge, ticket: HanaOperationTicket) { + do { + let closing = try JSONDecoder().decode(HanaCursorClosing.self, from: bridge.closeCursor(ticket)) + if closing.sessionLost { + markLost(ticket.session) + } + } catch { + if (error as? HanaBridgeFailure)?.kind == .connectionLost { + markLost(ticket.session) + } + Self.logger.error("SAP HANA cursor did not close: \(error.localizedDescription, privacy: .private)") + } + } + private func perform( _ target: Target, cancellation slot: HanaOperationSlot, diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift index 93afd8b101..337323aff8 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift @@ -152,6 +152,18 @@ final class HanaHelperBridge: HanaNativeBridge, @unchecked Sendable { try call(.explain, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } } + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data { + try call(.openCursor, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } + } + + func fetch(_ ticket: HanaOperationTicket) throws -> Data { + try call(.fetch, ticket: ticket) { HanaHelperMessage.operation($0) } + } + + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data { + try call(.closeCursor, ticket: ticket) { HanaHelperMessage.operation($0) } + } + func ping(_ ticket: HanaOperationTicket) throws { _ = try call(.ping, ticket: ticket) { HanaHelperMessage.operation($0) } } diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift index 74f17e800d..764e5d2605 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift @@ -8,6 +8,9 @@ enum HanaHelperOpcode: UInt8, Sendable { case ping = 5 case cancel = 6 case close = 7 + case openCursor = 8 + case fetch = 9 + case closeCursor = 10 } enum HanaHelperFrameError: Error, Equatable, Sendable { @@ -135,7 +138,7 @@ struct HanaHelperGreeting: Equatable, Sendable { } enum HanaHelperHandshake { - static let protocolVersion = 1 + static let protocolVersion = 2 static let frameID: UInt64 = 0 static let maximumBodyLength = 4_096 static let acceptedForcedSeverGraceSeconds = 1...3_600 diff --git a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift index 7e72d072b1..ec789006e5 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift @@ -5,6 +5,9 @@ protocol HanaNativeBridge: Sendable { func connect(_ ticket: HanaOperationTicket) throws -> Data func execute(_ ticket: HanaOperationTicket, request: Data) throws -> Data func explain(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func fetch(_ ticket: HanaOperationTicket) throws -> Data + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data func ping(_ ticket: HanaOperationTicket) throws func cancel(_ ticket: HanaOperationTicket) func close(session: UInt64) diff --git a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift index 6bc959cc7c..571d6669e2 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift @@ -100,6 +100,22 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try await run(query, parameters: parameters, rowCap: nil) } + // Collected here rather than through boundedQueryFromStream, which would drop the column hints and the + // large object warning that execute returns. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + let report = HanaStreamReport() + let collected = try await PluginBoundedStream.collect( + stream(query, report: report), + rowCap: rowCap, + startedAt: Date() + ) + return report.completing(collected) + } + + func streamRows(query: String) -> AsyncThrowingStream { + stream(query, report: HanaStreamReport()) + } + func switchSchema(to schema: String) async throws { guard !schema.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw HanaError(kind: .configuration, message: String(localized: "The SAP HANA schema name cannot be empty.")) @@ -159,6 +175,63 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + private func stream(_ sql: String, report: HanaStreamReport) -> AsyncThrowingStream { + PluginRowStream.make { continuation, abort in + let driver = self + let producer = Task { + do { + try await driver.produce(sql, into: continuation, report: report) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + report.logTruncatedLargeObjects() + } + // The cancel takes the Stop path: the helper ends a fetch in flight, or the cursor loop closes the cursor. + abort.onAbort { producer.cancel() } + } + } + + private func produce( + _ sql: String, + into continuation: AsyncThrowingStream.Continuation, + report: HanaStreamReport + ) async throws { + if HanaExplainStatement.explainedStatement(in: sql) != nil { + let result = try await run(sql, parameters: nil, rowCap: nil) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + ))) + if !result.rows.isEmpty { + continuation.yield(.rows(result.rows)) + } + return + } + let query = beginQuery() + defer { endQuery(query) } + do { + try await session.stream( + sql: sql, + cancellation: query.cancellation, + onHeader: { header in + report.record(header) + continuation.yield(.header(PluginStreamHeader( + columns: header.columns, + columnTypeNames: header.columnTypeNames + ))) + }, + onBatch: { batch in + report.record(truncatedLargeObjects: batch.truncatedLobCount) + continuation.yield(.rows(HanaResultMapping.pluginRows(batch.rows))) + } + ) + } catch let failure as HanaBridgeFailure { + let requested = Task.isCancelled || cancellationGate.isCancelled(query.generation) + throw HanaFailureMapping.error(for: failure, cancellationRequested: requested) + } + } + private func beginQuery() -> RunningQuery { let cancellation = HanaOperationSlot() return stateLock.withLock { diff --git a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift index 2da12083d7..0b2131cc46 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift @@ -1,4 +1,5 @@ import Foundation +import os import TableProPluginKit enum HanaResultMapping { @@ -6,15 +7,23 @@ enum HanaResultMapping { PluginQueryResult( columns: envelope.columns, columnTypeNames: envelope.columnTypeNames, - rows: envelope.rows.map { row in row.map(\.pluginValue) }, + rows: pluginRows(envelope.rows), rowsAffected: Int(clamping: envelope.rowsAffected), timing: PluginQueryTiming(total: envelope.executionTime), isTruncated: envelope.isTruncated, statusMessage: statusMessage(truncatedLobCount: envelope.truncatedLobCount), - columnMeta: columnMeta(for: envelope) + columnMeta: columnMeta( + columns: envelope.columns, + columnTypeNames: envelope.columnTypeNames, + classifications: envelope.columnClassifications + ) ) } + static func pluginRows(_ rows: [[HanaBridgeCell]]) -> [PluginRow] { + rows.map { row in row.map(\.pluginValue) } + } + static func statusMessage(truncatedLobCount: Int) -> String? { guard truncatedLobCount > 0 else { return nil } return String( @@ -23,17 +32,20 @@ enum HanaResultMapping { ) } - static func columnMeta(for envelope: HanaResultEnvelope) -> [PluginColumnInfo]? { - let hints = envelope.columnClassifications + static func columnMeta( + columns: [String], + columnTypeNames: [String], + classifications hints: [String?] + ) -> [PluginColumnInfo]? { guard hints.contains(where: { $0 != nil }), - hints.count == envelope.columns.count, - envelope.columnTypeNames.count == envelope.columns.count else { + hints.count == columns.count, + columnTypeNames.count == columns.count else { return nil } - return envelope.columns.indices.map { index in + return columns.indices.map { index in PluginColumnInfo( - name: envelope.columns[index], - dataType: envelope.columnTypeNames[index], + name: columns[index], + dataType: columnTypeNames[index], isNullable: true, isPrimaryKey: false, defaultValue: nil, @@ -55,3 +67,49 @@ enum HanaResultMapping { } } } + +final class HanaStreamReport: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "HanaPluginDriver") + + private let lock = NSLock() + private var header: HanaCursorHeader? + private var truncatedLargeObjects = 0 + + func record(_ header: HanaCursorHeader) { + lock.withLock { self.header = header } + } + + func record(truncatedLargeObjects count: Int) { + guard count > 0 else { return } + lock.withLock { truncatedLargeObjects += count } + } + + // A stream header has no field for the warning a result carries in statusMessage, so an export only logs it. + func logTruncatedLargeObjects() { + let count = lock.withLock { truncatedLargeObjects } + guard count > 0 else { return } + Self.logger.warning("SAP HANA cut \(count) large object values in a streamed result at 64 MiB") + } + + func completing(_ collected: PluginQueryResult) -> PluginQueryResult { + let (header, count) = lock.withLock { (self.header, self.truncatedLargeObjects) } + var result = PluginQueryResult( + columns: collected.columns, + columnTypeNames: collected.columnTypeNames, + rows: collected.rows, + rowsAffected: collected.rowsAffected, + timing: collected.timing, + isTruncated: collected.isTruncated, + statusMessage: HanaResultMapping.statusMessage(truncatedLobCount: count), + columnMeta: header.flatMap { + HanaResultMapping.columnMeta( + columns: $0.columns, + columnTypeNames: $0.columnTypeNames, + classifications: $0.columnClassifications + ) + } + ) + result.columnOrigins = collected.columnOrigins + return result + } +} diff --git a/apple/Plugins/HanaDriverPlugin/HanaSession.swift b/apple/Plugins/HanaDriverPlugin/HanaSession.swift index 3efa43e337..7405b5be75 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaSession.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaSession.swift @@ -13,6 +13,12 @@ protocol HanaSession: AnyObject, Sendable { cancellation: HanaOperationSlot ) async throws -> HanaResultEnvelope func explain(sql: String, cancellation: HanaOperationSlot) async throws -> HanaResultEnvelope + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws func cancel(_ cancellation: HanaOperationSlot) func applyQueryTimeout(seconds: Int) } diff --git a/apple/Plugins/HanaDriverPlugin/README.md b/apple/Plugins/HanaDriverPlugin/README.md index 0a7efd020c..7834573d17 100644 --- a/apple/Plugins/HanaDriverPlugin/README.md +++ b/apple/Plugins/HanaDriverPlugin/README.md @@ -11,7 +11,8 @@ What the plugin does: - User name and password login, with TLS in every TablePro mode. Preferred and Required encrypt without checking the certificate, Verify CA checks the chain only, and Verify Identity checks the chain and the host name, against the system roots unless a CA file is set. A client certificate and key can be added for mutual TLS. The TLS Server Name field overrides the name checked by Verify Identity when the certificate names another host. - Schema browsing, columns with full types, primary keys, identity and generated columns, indexes, foreign keys, row counts, table DDL and view definitions. - SQL execution with bound parameters, so grid edits, inserts and deletes work. +- Exports and capped editor reads stream. The helper keeps the statement's result open as a cursor and sends it in batches of up to 1,000 rows or 4 MiB, so the row count has no limit. The query timeout covers running the statement and stops once its result is ready. A stream that ends early, or Stop after the result is ready, ends the fetch at its next row and closes the result without `ALTER SYSTEM CANCEL SESSION`. - Stop and the query timeout. Both send `ALTER SYSTEM CANCEL SESSION` for the connection's own session, which keeps the session open. The bridge closes the session instead, and TablePro reconnects, when the server refuses the cancel or does not answer it within 10 seconds, or when the statement is still running `hana.ForcedSeverGrace` (30 seconds) later. After Stop, and not after a timeout, the plugin also stops the helper when the operation is still pending 10 seconds past that grace, which the helper announces as `forcedSeverGraceSeconds` in its first frame. - `EXPLAIN PLAN FOR` runs as one step that saves the plan, reads it and deletes it again. Reading plans needs the `OPTIMIZER ADMIN` privilege on recent HANA versions. -Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short, and the result says so. A reply frame holds at most 2 GiB - 1 bytes, so a larger result comes back as an internal error, "the result is larger than 2 GiB". +Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short. A query result says so; an export only logs it, since a stream has no field for the warning. A reply frame holds at most 2 GiB - 1 bytes, so a statement that is not streamed and returns more comes back as an internal error, "the result is larger than 2 GiB". diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift index 1f73cebab5..9e670008b7 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift @@ -63,17 +63,7 @@ internal final class InfluxDBConnection: @unchecked Sendable { } func send(_ request: InfluxRequest, timeout: TimeInterval? = nil) async throws -> InfluxDBResponse { - guard let url = request.url(base: baseURL) else { - throw InfluxError.invalidURL(baseURL.absoluteString + request.path) - } - var urlRequest = URLRequest(url: url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - + let urlRequest = try makeURLRequest(request, timeout: timeout) let taskBox = PluginURLSessionTaskBox() let key = ObjectIdentifier(taskBox) lock.withLock { inFlight[key] = taskBox } @@ -115,6 +105,48 @@ internal final class InfluxDBConnection: @unchecked Sendable { task.resume() } } + + // The body chunk by chunk, the next one read only once `body` returns. A status outside 2xx throws what `send`'s + // caller would, and `cancelAll` stops it like any other request. + func stream(_ request: InfluxRequest, body: (Data) async throws -> Void) async throws { + let urlRequest = try makeURLRequest(request, timeout: nil) + let taskBox = PluginURLSessionTaskBox() + let key = ObjectIdentifier(taskBox) + lock.withLock { inFlight[key] = taskBox } + defer { lock.withLock { _ = inFlight.removeValue(forKey: key) } } + // Made under the lock `close()` invalidates with, as in `perform`. + guard let task = lock.withLock({ session?.dataTask(with: urlRequest) }) else { + throw InfluxError.notConnected + } + let chunks = InfluxDBHTTPChunks(task: task) + taskBox.set(task) + task.resume() + try await withTaskCancellationHandler { + do { + try await chunks.forEachChunk(body) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } + // A cancelled reader ends the chunk sequence without an error. + try Task.checkCancellation() + } onCancel: { + taskBox.cancel() + } + } + + private func makeURLRequest(_ request: InfluxRequest, timeout: TimeInterval?) throws -> URLRequest { + guard let url = request.url(base: baseURL) else { + throw InfluxError.invalidURL(baseURL.absoluteString + request.path) + } + var urlRequest = URLRequest(url: url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + return urlRequest + } } private final class InfluxDBTrustingDelegate: PluginRedirectRefusingDelegate, @unchecked Sendable { diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift new file mode 100644 index 0000000000..b492c1455f --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift @@ -0,0 +1,109 @@ +import Foundation +import TableProInfluxDBCore + +// The body of a streamed read as the socket delivers it. It is the task's delegate, not the session's, so the +// session's own delegate still answers the server trust challenge. The transfer is suspended as each chunk is handed +// over and resumed once the decoder is done with it, so undecoded body never piles up. +final class InfluxDBHTTPChunks: PluginRedirectRefusingDelegate, URLSessionDataDelegate, @unchecked Sendable { + // An error body is a message for the user; InfluxDB's run to a few hundred bytes. + private static let errorBodyByteCap = 65_536 + + private let stream: AsyncThrowingStream + private let continuation: AsyncThrowingStream.Continuation + private let lock = NSLock() + private var task: URLSessionDataTask? + private var isTransferSuspended = false + private var refusal: Error? + private var failedResponse: HTTPURLResponse? + private var failureBody = Data() + + // Takes over the task's delegate; the caller resumes it. + init(task: URLSessionDataTask) { + (stream, continuation) = AsyncThrowingStream.makeStream() + self.task = task + super.init() + task.delegate = self + continuation.onTermination = { _ in task.cancel() } + } + + // The task is torn down on every way out. A reader that throws leaves it suspended mid-body with nothing left to + // resume it, which would hold the connection open until the driver disconnects. + func forEachChunk(_ body: (Data) async throws -> Void) async throws { + defer { endTransfer() } + for try await chunk in stream { + try await body(chunk) + resumeTransfer() + } + } + + func urlSession( + _ session: URLSession, + dataTask: URLSessionDataTask, + didReceive response: URLResponse, + completionHandler: @escaping (URLSession.ResponseDisposition) -> Void + ) { + guard let http = response as? HTTPURLResponse else { + lock.withLock { refusal = InfluxError.malformedResponse("Not an HTTP response") } + completionHandler(.cancel) + return + } + if !(200 ..< 300).contains(http.statusCode) { + lock.withLock { failedResponse = http } + } + completionHandler(.allow) + } + + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + let isFailure = lock.withLock { () -> Bool in + guard failedResponse != nil else { return false } + if failureBody.count < Self.errorBodyByteCap { + failureBody.append(data.prefix(Self.errorBodyByteCap - failureBody.count)) + } + return true + } + guard !isFailure else { return } + + // Suspended under the lock and before the reader sees the chunk, so the flag and the task never disagree. + lock.withLock { + guard !isTransferSuspended else { return } + isTransferSuspended = true + dataTask.suspend() + } + continuation.yield(data) + } + + // The same error `send` gives for the same response, so a refused statement reads alike on both paths. + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { + let failure = lock.withLock { () -> Error? in + if let refusal { return refusal } + if let error { return error } + guard let response = failedResponse else { return nil } + return InfluxDBResponse(statusCode: response.statusCode, headers: response.allHeaderFields, data: failureBody) + .error + } + guard let failure else { + continuation.finish() + return + } + continuation.finish(throwing: failure) + } + + private func resumeTransfer() { + lock.withLock { + guard isTransferSuspended else { return } + isTransferSuspended = false + task?.resume() + } + } + + private func endTransfer() { + let outstanding = lock.withLock { () -> URLSessionDataTask? in + let current = task + task = nil + isTransferSuspended = false + return current + } + outstanding?.cancel() + continuation.finish() + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift new file mode 100644 index 0000000000..9bf9af67a4 --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift @@ -0,0 +1,106 @@ +import Foundation +import TableProInfluxDBCore +import TableProPluginKit + +// Only the grid's own pages and exports stream: their columns are the measurement's, known before the first row. Flux +// and editor statements keep the collected read, whose columns are the union of everything that comes back. +extension InfluxDBPluginDriver { + func streamRows(query: String) -> AsyncThrowingStream { + guard let remembered = streamedGrid(query) else { return collectedStream(query) } + return PluginRowStream.make { continuation, abort in + let producer = Task { + do { + try await self.streamGrid(remembered, abort: abort, continuation: continuation) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } + + // A stream carries no column metadata, so the classification `execute` gives each column is added back. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + guard streamedGrid(query) != nil else { return nil } + let result = try await boundedQueryFromStream(query: query, rowCap: rowCap) + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: result.rows, + rowsAffected: 0, + timing: result.timing, + isTruncated: result.isTruncated, + columnMeta: Self.columnMeta(columns: result.columns, typeNames: result.columnTypeNames) + ) + } + + private func streamedGrid(_ query: String) -> RememberedGridQuery? { + guard let remembered = rememberedGrid(query), remembered.query.shape != .count else { return nil } + return remembered + } + + private func streamGrid( + _ remembered: RememberedGridQuery, + abort: PluginStreamAbort, + continuation: AsyncThrowingStream.Continuation + ) async throws { + guard !abort.isAborted else { throw CancellationError() } + let (_, server) = try requireConnection() + let (statement, schema) = try await gridStatementFromSchema( + remembered.query, + hostKinds: remembered.hostKinds, + generation: server.generation + ) + let (connection, current, request) = try await queryRequest(statement, streamed: true) + guard !abort.isAborted else { throw CancellationError() } + + var decoder = InfluxGridStream( + body: current.generation == .v3 ? .csv : .influxQLChunks, + grid: remembered.query, + schema: schema + ) + var headerSent = false + func send(_ batches: [InfluxResultTable]) { + for batch in batches { + if !headerSent { + headerSent = true + continuation.yield(.header(PluginStreamHeader(columns: batch.columns, columnTypeNames: batch.typeNames))) + } + if !batch.rows.isEmpty { + continuation.yield(.rows(Self.cells(batch.rows))) + } + } + } + try await connection.stream(request) { chunk in + guard !abort.isAborted else { throw CancellationError() } + let batches = try decoder.consume(chunk) + send(batches) + } + let rest = try decoder.finish() + send(rest) + } + + // PluginKit's default stream, which the override above replaces for every statement. + private func collectedStream(_ query: String) -> AsyncThrowingStream { + PluginRowStream.make { continuation, abort in + let producer = Task { + do { + try Task.checkCancellation() + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + ))) + if !result.rows.isEmpty { + continuation.yield(.rows(result.rows)) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift index 99a420548f..e2bc432c05 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift @@ -28,7 +28,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send var serverVersion: String? { lock.withLock { server?.version } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var currentDatabase: String? { lock.withLock { database.isEmpty ? nil : database } } func beginTransaction() async throws {} @@ -152,7 +152,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send let started = Date() let (_, server) = try requireConnection() let table: InfluxResultTable - if let remembered = lock.withLock({ gridQueries[query] }) { + if let remembered = rememberedGrid(query) { table = try await runGrid(remembered.query, hostKinds: remembered.hostKinds, generation: server.generation) } else { table = try await run(InfluxStatement.route(query, generation: server.generation)) @@ -160,22 +160,30 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send return PluginQueryResult( columns: table.columns, columnTypeNames: table.typeNames, - rows: table.rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } }, + rows: Self.cells(table.rows), rowsAffected: 0, executionTime: Date().timeIntervalSince(started), - columnMeta: zip(table.columns, table.typeNames).map { name, typeName in - Self.columnInfo(name: name, typeName: typeName, role: nil) - } + columnMeta: Self.columnMeta(columns: table.columns, typeNames: table.typeNames) ) } - // A grid statement built before the measurement's columns were known is written again from them here. - // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. private func runGrid( _ grid: InfluxGridQuery, hostKinds: [String: InfluxColumnKind], generation: InfluxGeneration ) async throws -> InfluxResultTable { + let (statement, schema) = try await gridStatementFromSchema(grid, hostKinds: hostKinds, generation: generation) + let table = try await run(statement).typed(by: schema) + return table.columns.isEmpty ? grid.emptyResult(schema: schema) : table + } + + // A grid statement built before the measurement's columns were known is written again from them here. + // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. + func gridStatementFromSchema( + _ grid: InfluxGridQuery, + hostKinds: [String: InfluxColumnKind], + generation: InfluxGeneration + ) async throws -> (statement: InfluxStatement, schema: [InfluxColumn]) { var schema = try await columns(of: grid.measurement) let known = Set(schema.map(\.name)) if grid.filters.contains(where: { !known.contains($0.column) }) { @@ -186,20 +194,31 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send guard let text = grid.statement(language: language, kinds: kinds, marked: false) else { throw InfluxError.filterNotExpressible(language) } - var table = try await run(InfluxStatement(language: language, text: text)).typed(by: schema) - if table.columns.isEmpty { - let names = grid.projection.isEmpty ? schema.map(\.name) : grid.projection - let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) - table = InfluxResultTable( - columns: names, - typeNames: names.map { types[$0] ?? InfluxTypeName.string }, - rows: [] - ) - } - return table + return (InfluxStatement(language: language, text: text), schema) + } + + func rememberedGrid(_ statement: String) -> RememberedGridQuery? { + lock.withLock { gridQueries[statement] } } func run(_ statement: InfluxStatement) async throws -> InfluxResultTable { + let (connection, server, request) = try await queryRequest(statement) + let response = try await connection.send(request) + guard response.isSuccess else { throw response.error } + switch (server.generation, statement.language) { + case (.v3, _): + return try InfluxResponseDecoder.decodeV3CSV(response.data) + case (_, .flux): + return try InfluxResponseDecoder.decodeFluxCSV(response.data) + default: + return try InfluxResponseDecoder.decodeV1JSON(response.data) + } + } + + func queryRequest( + _ statement: InfluxStatement, + streamed: Bool = false + ) async throws -> (connection: InfluxDBConnection, server: InfluxServerInfo, request: InfluxRequest) { let (connection, server) = try requireConnection() let (database, organization) = lock.withLock { (self.database, self.organization) } if server.generation == .v3, statement.language == .sql, database.isEmpty { @@ -221,18 +240,10 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send retentionPolicy: target.retentionPolicy, organization: organization, generation: server.generation, - credentials: connection.credentials + credentials: connection.credentials, + streamed: streamed ) - let response = try await connection.send(request) - guard response.isSuccess else { throw response.error } - switch (server.generation, statement.language) { - case (.v3, _): - return try InfluxResponseDecoder.decodeV3CSV(response.data) - case (_, .flux): - return try InfluxResponseDecoder.decodeFluxCSV(response.data) - default: - return try InfluxResponseDecoder.decodeV1JSON(response.data) - } + return (connection, server, request) } // Safe Mode read this script with TablePro's lexer. The server parses it first, and a script it reads as writing or @@ -360,10 +371,18 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send columnCheck: nil ) } + + static func columnMeta(columns: [String], typeNames: [String]) -> [PluginColumnInfo] { + zip(columns, typeNames).map { name, typeName in columnInfo(name: name, typeName: typeName, role: nil) } + } + + static func cells(_ rows: [[String?]]) -> [PluginRow] { + rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } } + } } // The statements the driver built for its grid, so `execute` runs them knowing the measurement they read. -private struct RememberedGridQuery { +struct RememberedGridQuery { let query: InfluxGridQuery let hostKinds: [String: InfluxColumnKind] } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index 0f78551a21..60fb612887 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -325,6 +325,7 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .uuidCellText, .userManagement, .temporaryTables, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift index a195f9ef45..d849f6674f 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift @@ -14,9 +14,6 @@ import TableProSQLQuoting /// second row that happens to look the same. That bound is only safe when every column takes part in the match: a /// match with one left out can pick a different row, so such a change is refused instead. struct MSSQLStatementGenerator { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - let qualifiedTable: String let columns: [String] let primaryKeyColumns: [String] @@ -38,7 +35,8 @@ struct MSSQLStatementGenerator { let statement = try insertStatement( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: @@ -72,10 +70,11 @@ struct MSSQLStatementGenerator { func insertStatement( values: [PluginCellValue], rowIndex: Int = 0, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) throws -> (statement: String, parameters: [PluginCellValue]) { let written = zip(columns, values).enumerated() - .filter { $0.element.1 != Self.defaultMarker } + .filter { !defaultColumns.contains($0.offset) } .map { (index: $0.offset, column: $0.element.0, value: $0.element.1) } if let owned = written.first(where: { context.serverOwnedColumns.contains($0.column) }) { throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: Self.serverOwnedReason(owned.column)) @@ -106,7 +105,7 @@ struct MSSQLStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = Self.quote(cellChange.columnName) - guard cellChange.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift index 272d58c088..cbbbd6569a 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -11,11 +11,16 @@ import TableProPluginKit private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sendable { typealias InboundIn = ByteBuffer + private let closed: NIOLockedValueBox private var reader: MemcachedReplyReader? private var continuation: CheckedContinuation? private var failure: Error? private let lock = NIOLock() + init(closed: NIOLockedValueBox) { + self.closed = closed + } + func expect( _ shape: MemcachedReplyShape, lineLimit: Int, @@ -57,6 +62,7 @@ private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sen } func channelInactive(context: ChannelHandlerContext) { + closed.withLockedValue { $0 = true } finish(with: MemcachedError.connectionClosed) } @@ -124,6 +130,10 @@ actor MemcachedConnection { private var wireSlotIsTaken = false private var waitingForWireSlot: [CheckedContinuation] = [] private var requestTimeout: TimeAmount? + private nonisolated let closed = NIOLockedValueBox(false) + + // True once the channel closed under the connection, as a cancelled or timed-out request closes it. + nonisolated var hasClosed: Bool { closed.withLockedValue { $0 } } /// A reply that has not finished by then closes the connection, since the text protocol cannot skip it. func setRequestTimeout(seconds: Int) { @@ -138,7 +148,7 @@ actor MemcachedConnection { guard channel == nil else { return } let sslContext = try Self.makeSSLContext(endpoint.ssl) let serverHostname = endpoint.ssl.verifiesHostname ? endpoint.tlsServerName : nil - let replyHandler = MemcachedReplyHandler() + let replyHandler = MemcachedReplyHandler(closed: closed) let bootstrap = ClientBootstrap(group: NIOSingletons.posixEventLoopGroup) .connectTimeout(endpoint.connectTimeout) .channelOption(ChannelOptions.socketOption(.so_keepalive), value: 1) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index cc1021e399..93d1cda577 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -13,6 +13,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// Keys one listing reads before it stops, so a cache holding millions of items cannot exhaust memory. static let listingLimit = 1_000_000 private static let valuesPerRequest = 100 + private static let rowsPerStreamBatch = 1_000 private let endpoint: MemcachedEndpoint private let tunneledServerName: String? @@ -40,7 +41,10 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var serverVersion: String? { lock.withLock { _serverVersion } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [] } + var capabilities: PluginCapabilities { [.columnDefaultMarks] } + + // A stopped request closes the channel, since the text protocol cannot skip its reply; the host then reconnects. + var hasLostConnection: Bool { lock.withLock { connection }?.hasClosed ?? false } func beginTransaction() async throws {} func commitTransaction() async throws {} @@ -208,24 +212,28 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Browsing + private struct BrowsePage { + let entries: [MemcachedKeyEntry] + let serverTime: Int64 + // The listing stopped at `listingLimit`, so the page is drawn from part of the cache. + let isCut: Bool + } + private func browse(_ spec: MemcachedBrowseSpec, started: Date) async throws -> PluginQueryResult { let connection = try liveConnection() - let serverTime = try await Self.serverTime(on: connection) - let listed = try await listKeys().filter { !$0.isExpired(at: serverTime) } - let page = try spec.page(of: listed) - - var values: [String: MemcachedValue] = [:] - var start = page.startIndex - while start < page.endIndex { - try Task.checkCancellation() - let end = min(start + Self.valuesPerRequest, page.endIndex) - let keys = page[start ..< end].map(\.key) - if case .values(let read) = try await connection.send(.get(keys: keys, withCAS: false)) { - for value in read { values[value.key] = value } - } - start = end - } + let page = try await browsePage(spec, on: connection) + let rows = try await rows(of: page.entries, serverTime: page.serverTime, on: connection) + return result( + columns: MemcachedColumn.allCases.map(\.rawValue), + types: MemcachedColumn.allCases.map(\.typeName), + rows: rows, started: started + ) + } + private func browsePage(_ spec: MemcachedBrowseSpec, on connection: MemcachedConnection) async throws -> BrowsePage { + let serverTime = try await Self.serverTime(on: connection) + let listing = try await listKeys() + let page = try spec.page(of: listing.entries.filter { !$0.isExpired(at: serverTime) }) lock.withLock { serverClockOffset = MemcachedServerClock.offset( serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) @@ -233,8 +241,27 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } } + return BrowsePage(entries: page, serverTime: serverTime, isCut: listing.isCut) + } - let rows: [PluginRow] = page.compactMap { entry in + // A key removed since the listing has no value and no row. + private func rows( + of entries: some Collection, + serverTime: Int64, + on connection: MemcachedConnection + ) async throws -> [PluginRow] { + var values: [String: MemcachedValue] = [:] + let keys = entries.map(\.key) + var start = keys.startIndex + while start < keys.endIndex { + try Task.checkCancellation() + let end = min(start + Self.valuesPerRequest, keys.endIndex) + if case .values(let read) = try await connection.send(.get(keys: Array(keys[start ..< end]), withCAS: false)) { + for value in read { values[value.key] = value } + } + start = end + } + return entries.compactMap { entry in guard let value = values[entry.key] else { return nil } return [ .text(entry.key), @@ -243,20 +270,15 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .text(String(value.flags)), ] } - return result( - columns: MemcachedColumn.allCases.map(\.rawValue), - types: MemcachedColumn.allCases.map(\.typeName), - rows: rows, started: started - ) } - private func listKeys() async throws -> [MemcachedKeyEntry] { + private func listKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { if lock.withLock({ keySource }) == .metadump { do { return try await withListingConnection { connection in let reply = try await connection.send(.metadump(selector: "all"), lineLimit: Self.listingLimit) - guard case .lines(let lines, _) = reply else { return [] } - return lines.compactMap(MemcachedKeyListing.metadumpEntry) + guard case .lines(let lines, let truncated) = reply else { return ([], false) } + return (lines.compactMap(MemcachedKeyListing.metadumpEntry), truncated) } } catch let error as MemcachedServerError where error.kind != .busy && !error.needsAuthentication { Self.logger.info("metadump refused, listing with cachedump") @@ -266,18 +288,69 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return try await cachedumpKeys() } - private func cachedumpKeys() async throws -> [MemcachedKeyEntry] { + private func cachedumpKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { let connection = try liveConnection() - guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return [] } + guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return ([], false) } var entries: [MemcachedKeyEntry] = [] for slab in MemcachedKeyListing.slabClasses(fromItemStats: itemStats) { try Task.checkCancellation() let reply = try await connection.send(.stats(arguments: ["cachedump", String(slab), "0"])) guard case .lines(let lines, _) = reply else { continue } entries += lines.compactMap(MemcachedKeyListing.cachedumpEntry) - if entries.count >= Self.listingLimit { break } + if entries.count >= Self.listingLimit { return (Array(entries.prefix(Self.listingLimit)), true) } + } + return (entries, false) + } + + // MARK: - Streaming + + // An export reads every key, so its values are read and sent a batch at a time on a connection of their own, and + // stopping closes that connection, never the session's. + func streamRows(query: String) -> AsyncThrowingStream { + let statement = query.trimmingCharacters(in: .whitespacesAndNewlines) + let spec = MemcachedBrowseSpec.refusalMessage(statement) == nil ? MemcachedBrowseSpec.parse(statement) : nil + return PluginRowStream.make { continuation, abort in + let task = Task { + do { + if let spec { + try await self.streamBrowse(spec, into: continuation) + } else { + let result = try await self.execute(query: query) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: MemcachedError.wrap(error)) + } + } + abort.onAbort { task.cancel() } + } + } + + private func streamBrowse( + _ spec: MemcachedBrowseSpec, + into continuation: AsyncThrowingStream.Continuation + ) async throws { + let page = try await browsePage(spec, on: try liveConnection()) + var header = PluginStreamHeader( + columns: MemcachedColumn.allCases.map(\.rawValue), + columnTypeNames: MemcachedColumn.allCases.map(\.typeName) + ) + if page.isCut { header.rowLimit = page.entries.count } + continuation.yield(.header(header)) + try await withListingConnection { connection in + var start = page.entries.startIndex + while start < page.entries.endIndex { + try Task.checkCancellation() + let end = min(start + Self.rowsPerStreamBatch, page.entries.endIndex) + let rows = try await self.rows(of: page.entries[start ..< end], serverTime: page.serverTime, on: connection) + if !rows.isEmpty { continuation.yield(.rows(rows)) } + start = end + } } - return entries } /// Opens a connection for one listing and closes it after, even when the listing stopped early with the rest @@ -431,6 +504,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let offset = lock.withLock { serverClockOffset } let expiries = lock.withLock { shownExpiries } let generator = MemcachedStatementGenerator( diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 05bfefbd69..88935569d4 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -35,7 +35,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se func quoteIdentifier(_ name: String) -> String { name } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } func defaultExportQuery(table: String) -> String? { diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index e3ff61ce49..574e17e26f 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -40,8 +40,6 @@ nonisolated struct MongoDBStatementGenerator { /// connection. var capabilities: () -> MongoDBCapabilities = { .unknown } - private static let defaultMarker = "__DEFAULT__" - /// Where a value written whole comes from. A cell carries only text or bytes, so this is all the /// generator knows about the type the value has to keep. private enum Provenance { @@ -133,7 +131,7 @@ nonisolated struct MongoDBStatementGenerator { let absent = change.absentColumns ?? [] let written = cells.filter { cell in let serverAssignsIdentity = cell.field == MongoDBCollectionDDL.idField && cell.value.isNull - return !serverAssignsIdentity && !isLeftOut(cell.value, isAbsent: absent.contains(cell.column)) + return !serverAssignsIdentity && !absent.contains(cell.column) && !change.isColumnDefault(cell.column) } let entries = try written.map { cell in let provenance: Provenance = typed[cell.field] == cell.value ? .typedIntoNewDocument : .copiedIntoNewDocument @@ -142,10 +140,6 @@ nonisolated struct MongoDBStatementGenerator { return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" } - private func isLeftOut(_ value: PluginCellValue, isAbsent: Bool) -> Bool { - isAbsent || value.asText == Self.defaultMarker - } - /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. @@ -187,7 +181,7 @@ nonisolated struct MongoDBStatementGenerator { let idField = MongoDBCollectionDDL.idField var entries = ["\(quotedKey(idField)): \(try Self.identityJson(locator: locator))"] for (index, value) in row.enumerated() where index != idIndex && index < columns.count { - guard !isLeftOut(value, isAbsent: absent.contains(index)) else { continue } + guard !absent.contains(index) else { continue } let field = columns[index] entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))") } @@ -226,7 +220,7 @@ nonisolated struct MongoDBStatementGenerator { let cellWrites = try change.cellChanges.map { try cellWrite( field: $0.columnName, from: $0.oldValue, to: $0.newValue, - removesField: removed.contains($0.columnIndex) + removesField: removed.contains($0.columnIndex), isDefault: change.isColumnDefault($0.columnIndex) ) } let update = try updateDocument(for: cellWrites) @@ -237,17 +231,18 @@ nonisolated struct MongoDBStatementGenerator { field: String, from oldValue: PluginCellValue, to newValue: PluginCellValue, - removesField: Bool + removesField: Bool, + isDefault: Bool ) throws -> CellWrite { guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged } guard !removesField else { return .remove(field: field) } + guard !isDefault else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } switch newValue { case .null: return .whole(field: field, json: "null") case .bytes(let data): return .whole(field: field, json: try binaryJson(data, field: field, provenance: .edit(replacing: oldValue))) case .text(let text): - guard text != Self.defaultMarker else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } if let nested = try nestedEdit(of: field, from: oldValue, to: text) { return nested } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 9500f5c6cd..f71b06fe00 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -215,6 +215,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .cellExpressions, .uuidCellText, .userManagement, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift index 13d859d092..5d9984ebe2 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift @@ -20,16 +20,21 @@ import TableProSQLQuoting /// because a NULL bind into an object-type column fails with ORA-00932; and text for an NCHAR, NVARCHAR2 or NCLOB /// column is `UNISTR`, because a literal or bind passes through the database character set (``OracleNationalText``). internal struct OracleRowWriter { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - /// A keyed delete carries at most this many key values. One statement with 32,767 binds took 30 seconds on 23ai /// and one with 1,000 took 0.07, and a statement that outlives the query timeout closes the session. static let maxValuesPerDelete = 1_000 static let dateMask = "YYYY-MM-DD HH24:MI:SS" static let timestampMask = "YYYY-MM-DD HH24:MI:SS.FF" - static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + // A cell reads `….ffffff+02:00`, the offset straight after the fraction, or `….ffffff Europe/Paris`, a region after + // a blank. TZH:TZM reads the first as written and TZR the second. + static let timestampWithOffsetMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FF TZR" + + static func timestampWithTimeZoneMask(for text: String?) -> String { + let zone = text?.split(separator: " ").last.flatMap(\.first) + return zone?.isLetter == true ? timestampWithTimeZoneMask : timestampWithOffsetMask + } let qualifiedTable: String let columns: [String] @@ -56,7 +61,8 @@ internal struct OracleRowWriter { let write = try insert( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) else { continue } try flushDeletes() writes.append(write) @@ -80,7 +86,8 @@ internal struct OracleRowWriter { private func insert( values: [PluginCellValue], rowIndex: Int, - expressionColumns: Set + expressionColumns: Set, + defaultColumns: Set ) throws -> PluginRowWrite? { guard let firstColumn = columns.first else { return nil } var names: [String] = [] @@ -89,7 +96,7 @@ internal struct OracleRowWriter { for (index, (column, value)) in zip(columns, values).enumerated() where !context.serverOwnedColumns.contains(column) { names.append(Self.quote(column)) - if value == Self.defaultMarker { + if defaultColumns.contains(index) { placeholders.append("DEFAULT") } else if let expression = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(expression) @@ -117,7 +124,7 @@ internal struct OracleRowWriter { var parameters: [PluginCellValue] = [] let assignments = try change.cellChanges.map { cell -> String in let column = Self.quote(cell.columnName) - guard cell.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cell.columnIndex) else { return "\(column) = DEFAULT" } if let expression = Self.expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) { return "\(column) = \(expression)" } @@ -261,10 +268,10 @@ internal struct OracleRowWriter { case .text(let text): if let literal = kind.inlineSQL(for: text) { return literal } if !kind.isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { - return kind.converting(quoted) + return kind.converting(quoted, text: text) } parameters.append(value) - return kind.converting("?") + return kind.converting("?", text: text) } } @@ -352,14 +359,14 @@ internal enum OracleValueKind: Equatable { /// `TO_TIMESTAMP` rejects (ORA-01830). `TO_TIMESTAMP_TZ` takes it, and text without an offset in the session's /// zone, so the value round-trips through SET and a keyless match whatever the session and database zones are /// (measured on 23ai with the session at `+07:00` and at `America/New_York` against a `+00:00` database). - func converting(_ operand: String) -> String { + func converting(_ operand: String, text: String? = nil) -> String { switch self { case .date: return "TO_DATE(\(operand), '\(OracleRowWriter.dateMask)')" case .timestamp: return "TO_TIMESTAMP(\(operand), '\(OracleRowWriter.timestampMask)')" case .timestampWithTimeZone, .timestampWithLocalTimeZone: - return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask)')" + return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask(for: text))')" case .number, .binaryFloat, .binaryDouble, .bfile, .nationalText, .nationalLOB, .other: return operand } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index ca822ad6e3..b8f4ad2a45 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -184,11 +184,16 @@ extension QdrantPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex) else { return nil } + // A cell left to the default has no text, so the point leaves it out and a defaulted id gets a new one. let cells: [QdrantCell] if let row = insertedRowData[change.rowIndex] { - cells = zip(columns, row).map { QdrantCell(column: $0, text: $1.asText) } + cells = zip(columns, row).enumerated().map { index, cell in + QdrantCell(column: cell.0, text: change.isColumnDefault(index) ? nil : cell.1.asText) + } } else { - cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + cells = change.cellChanges.map { + QdrantCell(column: $0.columnName, text: change.isColumnDefault($0.columnIndex) ? nil : $0.newValue.asText) + } } return QdrantRowChange(kind: .insert, rowIndex: change.rowIndex, id: nil, cells: cells) case .update: diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift index fe11918a98..90ea656972 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -21,7 +21,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 4d4ffcb41b..75ff9f44ec 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -63,7 +63,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - var supported: PluginCapabilities = [.truncateTable, .cancelQuery] + var supported: PluginCapabilities = [.truncateTable, .cancelQuery, .columnDefaultMarks] if redisConnection?.supportsTransactions ?? true { supported.insert(.transactions) } return supported } @@ -657,6 +657,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let partitioned = redisConnection?.partitionsKeyspace == true let generator = RedisStatementGenerator( namespaceName: table, diff --git a/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift new file mode 100644 index 0000000000..853eff5f75 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift @@ -0,0 +1,50 @@ +// +// RedisColumnDefaultTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct RedisColumnDefaultTests { + private let driver = RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + + private func valueEdit(to value: PluginCellValue, isDefault: Bool) -> PluginRowChange { + var change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "Value", oldValue: "old", newValue: value)], + originalRow: ["session:abc", "old", "-1"] + ) + if isDefault { change.defaultColumns = [1] } + return change + } + + private func writes(_ change: PluginRowChange) throws -> [String] { + try driver.generateRowWrites( + table: "", schema: nil, columns: ["Key", "Value", "TTL"], primaryKeyColumns: ["Key"], + changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + )?.map(\.statement) ?? [] + } + + @Test("The driver declares that it reads the default mark") + func declaresTheMark() { + #expect(driver.capabilities.contains(.columnDefaultMarks)) + } + + @Test("The text __DEFAULT__ is written as the value") + func defaultLookingTextIsAValue() throws { + let statements = try writes(valueEdit(to: "__DEFAULT__", isDefault: false)) + #expect(statements.contains { $0.contains("__DEFAULT__") }) + } + + @Test("A cell left to a default is refused, since Redis has none") + func markedDefaultIsRefused() { + #expect(throws: PluginRowWriteRefusal.self) { + _ = try writes(valueEdit(to: "__DEFAULT__", isDefault: true)) + } + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift index d4d5ca8870..13c8010726 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift @@ -43,6 +43,7 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .dataCompare, .rowWrites, .cellExpressions, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift index 5eac2b6580..1d72ede5f9 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift @@ -87,9 +87,6 @@ struct SnowflakeStatementGenerator { ) } - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func isSemiStructured(_ typeName: String) -> Bool { let base = typeName.uppercased().components(separatedBy: "(")[0].trimmingCharacters(in: .whitespaces) return ["VARIANT", "OBJECT", "ARRAY"].contains(base) @@ -100,18 +97,22 @@ struct SnowflakeStatementGenerator { insertedRowData: [Int: [PluginCellValue]] ) -> (statement: String, parameters: [PluginCellValue])? { var values: [(column: String, value: PluginCellValue)] = [] + var hasCells = false if let rowData = insertedRowData[change.rowIndex] { for (index, column) in columns.enumerated() where index < rowData.count { + hasCells = true + guard !change.isColumnDefault(index) else { continue } values.append((column, rowData[index])) } } else { for cellChange in change.cellChanges { + hasCells = true + guard !change.isColumnDefault(cellChange.columnIndex) else { continue } values.append((cellChange.columnName, cellChange.newValue)) } } - guard !values.isEmpty else { return nil } + guard hasCells else { return nil } let expressions = Set((change.expressionColumns ?? []).compactMap { columns.indices.contains($0) ? columns[$0] : nil }) - values.removeAll { $0.value == Self.defaultMarker } guard !values.isEmpty else { guard let firstColumn = columns.first else { return nil } return ("INSERT INTO \(qualifiedTable) (\(quoteIdentifier(firstColumn))) VALUES (DEFAULT)", []) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift index dc205a2a6a..5352ec4480 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift @@ -188,12 +188,13 @@ extension SpannerPluginDriver { SpannerRowChange.CellChange(column: $0.columnName, value: spannerCell($0.newValue)) }, originalRow: change.originalRow?.map(spannerCell), - expressionColumns: expressionColumnNames(of: change, columns: columns) + expressionColumns: columnNames(change.expressionColumns, of: change, columns: columns), + defaultColumns: columnNames(change.defaultColumns, of: change, columns: columns) ) } - private static func expressionColumnNames(of change: PluginRowChange, columns: [String]) -> Set { - guard let marked = change.expressionColumns, !marked.isEmpty else { return [] } + private static func columnNames(_ marked: Set?, of change: PluginRowChange, columns: [String]) -> Set { + guard let marked, !marked.isEmpty else { return [] } switch change.type { case .update: return Set(change.cellChanges.filter { marked.contains($0.columnIndex) }.map(\.columnName)) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift index cd645015f0..60f81f66fd 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift @@ -37,7 +37,10 @@ internal final class SpannerPluginDriver: PluginDatabaseDriver, @unchecked Senda } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites] + [ + .multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 1edab23077..fb2c57d7a3 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -38,7 +38,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema] + [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema, .columnDefaultMarks] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index c98626d2bd..4fe2be6314 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -8,13 +8,6 @@ import TableProNumberFormatting import TableProPluginKit public enum SurrealStatementGenerator { - static let autoIdMarker = "__DEFAULT__" - - static func isAutoDefault(_ value: PluginCellValue) -> Bool { - guard case let .text(text) = value else { return false } - return text.trimmingCharacters(in: .whitespaces) == autoIdMarker - } - public static func rowWrites( table: String, scope: SurrealScope, @@ -37,10 +30,15 @@ public enum SurrealStatementGenerator { writes.append(write) } + var defaults: [Int: Set] = [:] + for change in changes where change.type == .insert { + defaults[change.rowIndex, default: []].formUnion(change.defaultColumns ?? []) + } for index in insertedRowIndices.sorted() { guard let values = insertedRowData[index] else { continue } writes.append(try insert( - table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index + table: table, scope: scope, columns: columns, typing: typing, values: values, + defaults: defaults[index] ?? [], rowIndex: index )) } @@ -96,7 +94,7 @@ public enum SurrealStatementGenerator { } let recordKey = Self.recordText(columns: columns, originalRow: change.originalRow) guard let record = recordKey.flatMap({ SurrealQL.parseRecordId($0, fallbackTable: table) }) else { return nil } - let editable = change.cellChanges.filter { !Self.isAutoDefault($0.newValue) } + let editable = change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) } guard !editable.isEmpty else { return nil } var parameters: [PluginCellValue] = [SurrealCellCoder.parameter(.recordId(record))] @@ -126,6 +124,7 @@ public enum SurrealStatementGenerator { columns: [String], typing: FieldTyping, values: [PluginCellValue], + defaults: Set, rowIndex: Int ) throws(PluginRowWriteRefusal) -> PluginRowWrite { var parameters: [PluginCellValue] = [] @@ -133,13 +132,13 @@ public enum SurrealStatementGenerator { var target = SurrealQL.quoteIdentifier(table) for (index, column) in columns.enumerated() { - guard index < values.count else { continue } + guard index < values.count, !defaults.contains(index) else { continue } let cell = values[index] if column == SurrealInfoParser.recordIdColumn { guard case let .text(text) = cell else { continue } let trimmed = text.trimmingCharacters(in: .whitespaces) - guard !trimmed.isEmpty, trimmed != Self.autoIdMarker else { continue } + guard !trimmed.isEmpty else { continue } guard let record = SurrealQL.parseRecordId(text, fallbackTable: table) else { continue } parameters.append(SurrealCellCoder.parameter(.recordId(record))) target = "$p\(parameters.count - 1)" @@ -147,7 +146,6 @@ public enum SurrealStatementGenerator { } if case .null = cell { continue } - if Self.isAutoDefault(cell) { continue } try refuseShortened(cell, in: column, rowIndex: rowIndex) let value = typing.value(of: cell, in: column) parameters.append(SurrealCellCoder.parameter(value)) diff --git a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift index 702eaa1e56..1153452bb7 100644 --- a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift +++ b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift @@ -31,6 +31,7 @@ public enum PluginBoundedStream { var rows: [PluginRow] = [] rows.reserveCapacity(min(cap, 10_000)) var truncated = false + var declaredLimit: Int? /// The first batch to arrive is the first row the server produced, which is the only part /// of a streamed read that separates the query's own cost from the cost of moving its rows. var firstRowTime: TimeInterval? @@ -41,6 +42,7 @@ public enum PluginBoundedStream { columns = header.columns columnTypeNames = header.columnTypeNames columnOrigins = header.columnOrigins + declaredLimit = header.rowLimit case .rows(let batch): if firstRowTime == nil, !batch.isEmpty { firstRowTime = Date().timeIntervalSince(startedAt) @@ -55,6 +57,10 @@ public enum PluginBoundedStream { } if truncated { break } } + // A stream that ended at the limit its header declared was cut there, below the cap. + if let declaredLimit, rows.count >= declaredLimit { + truncated = true + } var result = PluginQueryResult( columns: columns, diff --git a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift index 59872dfaef..67ed3dcbf9 100644 --- a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift @@ -41,4 +41,7 @@ public struct PluginCapabilities: OptionSet, Sendable { // The driver refuses what `PluginStatementContext.allowsDestructive` and `allowsServerCode` forbid, for a language // the host cannot read in full. public static let statementPolicy = PluginCapabilities(rawValue: 1 << 22) + // The driver writes the column default only where `PluginRowChange.defaultColumns` marks one, so the text + // `__DEFAULT__` in any other cell is saved as text. + public static let columnDefaultMarks = PluginCapabilities(rawValue: 1 << 23) } diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 53d2af6929..0810eb1d63 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -70,6 +70,11 @@ public struct PluginRowChange: Sendable { /// that declares `PluginCapabilities.cellExpressions` is handed any. public var expressionColumns: Set? + // The columns whose new value is the column's default rather than a value: an update's `cellChanges`, or a new + // row's `insertedRowData` cells. Such a cell still holds the text `__DEFAULT__`, which a driver built before this + // field reads as the default. Every other cell is a value, that text included. + public var defaultColumns: Set? + public init( rowIndex: Int, type: ChangeType, @@ -81,6 +86,10 @@ public struct PluginRowChange: Sendable { self.cellChanges = cellChanges self.originalRow = originalRow } + + public func isColumnDefault(_ columnIndex: Int) -> Bool { + defaultColumns?.contains(columnIndex) ?? false + } } public protocol PluginDatabaseDriver: AnyObject, Sendable { @@ -1476,7 +1485,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } } @@ -1486,7 +1495,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeParameterized(query: query, parameters: parameters) } } @@ -1540,9 +1549,12 @@ public extension PluginDatabaseDriver { guard !parameters.isEmpty else { return try await execute(query: query) } + let kinds = PluginParameterBinder.hostParameterKinds let sql = try PluginParameterBinder.bind( query, - literals: parameters.map(sqlLiteral(for:)), + literals: parameters.enumerated().map { index, value in + sqlLiteral(for: value, kind: kinds.flatMap { $0.indices.contains(index) ? $0[index] : nil }) + }, style: parameterStyle, features: literalFeatures ) @@ -1550,11 +1562,17 @@ public extension PluginDatabaseDriver { } func sqlLiteral(for value: PluginCellValue) -> String { + sqlLiteral(for: value, kind: nil) + } + + // A nil kind is a value nobody typed, such as a grid cell, whose text the host wrote: number-shaped text is a number. + func sqlLiteral(for value: PluginCellValue, kind: PluginColumnKind?) -> String { switch value { case .null: return "NULL" case .text(let s): - return escapedParameterValue(s) + guard let kind else { return escapedParameterValue(s) } + return typedParameterValue(s, kind: kind) case .bytes(let data): var hex = "X'" hex.reserveCapacity(2 + data.count * 2 + 1) @@ -1578,6 +1596,23 @@ public extension PluginDatabaseDriver { PluginNumericLiteral.isValid(value) } + // Text the server cannot read as the declared number or boolean is sent as a string, for the server to refuse. + func typedParameterValue(_ value: String, kind: PluginColumnKind) -> String { + switch kind { + case .integer, .decimal: + if PluginSQLLiteral.isNumericLiteral(value, kind: kind) { return value } + case .boolean: + switch PluginSQLLiteral.booleanSynonym(for: value, kind: kind) { + case .isTrue: return "TRUE" + case .isFalse: return "FALSE" + case nil: break + } + case .text, .other: + break + } + return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) + } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { let raw: PluginQueryResult if let parameters { diff --git a/apple/Plugins/TableProPluginKit/PluginDriverError.swift b/apple/Plugins/TableProPluginKit/PluginDriverError.swift index 9581028e68..09b1da9da8 100644 --- a/apple/Plugins/TableProPluginKit/PluginDriverError.swift +++ b/apple/Plugins/TableProPluginKit/PluginDriverError.swift @@ -31,3 +31,17 @@ public extension PluginDriverError { return desc } } + +// A statement Stop ended, after which the server rolled back the transaction it ran in. A driver throws it in place of +// the cancellation, so the host can say what the Stop undid. +public struct PluginStopRolledBackTransaction: PluginDriverError { + public let pluginErrorCode: Int? + + public init(code: Int? = nil) { + pluginErrorCode = code + } + + public var pluginErrorMessage: String { + String(localized: "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone.") + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift index d2a06245da..1669cfdc11 100644 --- a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift +++ b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift @@ -20,6 +20,21 @@ public enum PluginParameterBinder { // The host's reading of the connection, set around a statement it hands to a driver. @TaskLocal static var hostFeatures: SQLLexicalFeatures? + @TaskLocal static var hostParameterKinds: [PluginColumnKind?]? + + // A driver whose context overload forwards to its context-free twin wraps the call in this, so the default binder + // still writes each value the way the host's context says. + public static func withHostContext( + _ context: PluginStatementContext, + operation: () async throws -> T + ) async rethrows -> T { + try await $hostFeatures.withValue(context.lexicalFeatures) { + try await $hostParameterKinds.withValue(context.parameterKinds) { + try await operation() + } + } + } + public static func bind( _ template: String, literals parameters: [String], diff --git a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift index 317bdad935..7e85f6a811 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift @@ -76,4 +76,13 @@ public struct PluginRowWriteRefusal: Error, LocalizedError, Sendable, Equatable } public var errorDescription: String? { reason } + + // For a driver whose engine has no column defaults: the first change that leaves a cell to one. + public static func columnDefaultsUnsupported(in changes: [PluginRowChange]) -> PluginRowWriteRefusal? { + guard let change = changes.first(where: { !($0.defaultColumns ?? []).isEmpty }) else { return nil } + return PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "This database has no column defaults, so a cell cannot be left to one. Enter a value.") + ) + } } diff --git a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift index 2af809f03b..ef39c06a8c 100644 --- a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift +++ b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift @@ -31,6 +31,10 @@ public struct PluginStatementContext: Sendable, Equatable { // that does not say. public var lexicalFeatures: SQLLexicalFeatures? + // The type the user gave each parameter, by position: `.text` and `.other` are written as strings, `.integer` and + // `.decimal` as numbers, `.boolean` as TRUE or FALSE. Nil, or a nil entry, where none was given. + public var parameterKinds: [PluginColumnKind?]? + public init() {} } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ea42a44623..7d851354e3 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -1,4 +1,5 @@ import Foundation +import TableProPluginKit import TableProTeradataCore final class TeradataAsyncConnection: @unchecked Sendable { @@ -30,6 +31,19 @@ final class TeradataAsyncConnection: @unchecked Sendable { try await run { try $0.execute(sql) } } + func execute(_ sql: String, rowCap: Int?) async throws -> TeradataCappedResult { + try await run { try $0.execute(sql, rowCap: rowCap) } + } + + // One queue item for the whole answer: the session holds one request, so no other call may run between its replies. + func stream( + _ sql: String, + onColumns: @escaping @Sendable ([TeradataColumn]) throws -> Void, + onRows: @escaping @Sendable ([[TeradataValue]]) throws -> Bool + ) async throws { + try await run { try $0.stream(sql, onColumns: onColumns, onRows: onRows) } + } + func disconnect() { queue.sync { connection.disconnect() } } @@ -55,6 +69,9 @@ final class TeradataAsyncConnection: @unchecked Sendable { let value = try body(self.connection) try cancellation.throwIfCancelled() continuation.resume(returning: value) + } catch TeradataWireError.rolledBackByStop { + // Kept past the cancel, so the host can say what the Stop undid. + continuation.resume(throwing: PluginStopRolledBackTransaction(code: TeradataConnection.userAbortCode)) } catch { continuation.resume(throwing: cancellation.isCancelled ? CancellationError() : error) } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index 6790d03b9d..c9dd03b96e 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -15,15 +15,19 @@ extension TeradataValue { } } +extension TeradataColumn { + var pluginTypeName: String { TeradataColumnType.wireTypeName(typeCode) } +} + extension TeradataResultSet { - func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { + func toPluginResult(executionTime: TimeInterval, isTruncated: Bool = false) -> PluginQueryResult { PluginQueryResult( columns: columns.map { $0.name }, - columnTypeNames: columns.map { TeradataColumnType.wireTypeName($0.typeCode) }, + columnTypeNames: columns.map { $0.pluginTypeName }, rows: rows.map { row in row.map { $0.asPluginCell } }, rowsAffected: activityCount, executionTime: executionTime, - isTruncated: false + isTruncated: isTruncated ) } } @@ -136,7 +140,10 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { true } var capabilities: PluginCapabilities { - [.cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites] + [ + .cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } init(config: DriverConnectionConfig) { @@ -203,27 +210,72 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func execute(query: String) async throws -> PluginQueryResult { - guard let connection else { throw TeradataWireError.connectionFailed("not connected") } - let start = Date() - let result = try await connection.execute(query) - return result.toPluginResult(executionTime: Date().timeIntervalSince(start)) + try await execute(query, rowCap: nil) } func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { - guard let parameters else { return try await execute(query: query) } - return try await executeParameterized(query: query, parameters: parameters) + try await execute(bound(query, parameters ?? []), rowCap: rowCap) + } + + // Not the stream: a Stop cancels the reading task, which ends a stream before a rollback error can cross it. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await execute(query, rowCap: rowCap) } - // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - guard !parameters.isEmpty else { return try await execute(query: query) } - let bound = try PluginParameterBinder.bind( + try await execute(bound(query, parameters), rowCap: nil) + } + + func streamRows(query: String) -> AsyncThrowingStream { + let connection = self.connection + return PluginRowStream.make { continuation, abort in + let task = Task { + guard let connection else { + continuation.finish(throwing: TeradataWireError.connectionFailed("not connected")) + return + } + do { + // Each reply's rows go out as it arrives, before the next one is asked for. + try await connection.stream( + query, + onColumns: { columns in + continuation.yield(.header(PluginStreamHeader( + columns: columns.map { $0.name }, + columnTypeNames: columns.map { $0.pluginTypeName } + ))) + }, + onRows: { rows in + guard !abort.isAborted else { return false } + continuation.yield(.rows(rows.map { row in row.map { $0.asPluginCell } })) + return true + } + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { task.cancel() } + } + } + + // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. + private func bound(_ query: String, _ parameters: [PluginCellValue]) throws -> String { + guard !parameters.isEmpty else { return query } + return try PluginParameterBinder.bind( query, literals: parameters.map(TeradataRowEditSQL.literal), style: .questionMark, features: [] ) - return try await execute(query: bound) + } + + private func execute(_ query: String, rowCap: Int?) async throws -> PluginQueryResult { + guard let connection else { throw TeradataWireError.connectionFailed("not connected") } + let start = Date() + let capped = try await connection.execute(query, rowCap: rowCap) + return capped.resultSet.toPluginResult( + executionTime: Date().timeIntervalSince(start), isTruncated: capped.isTruncated) } func cancelQuery() throws { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift index e9c842a0f7..a60e747208 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift @@ -36,7 +36,8 @@ extension TeradataPluginDriver { target: target, columns: columns, values: values, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) statements.append((insert, [])) case .update: @@ -45,7 +46,8 @@ extension TeradataPluginDriver { TeradataRowEditSQL.assignment( column: $0.columnName, value: $0.newValue, - isExpression: expressionColumns.contains($0.columnIndex) + isExpression: expressionColumns.contains($0.columnIndex), + isDefault: change.isColumnDefault($0.columnIndex) ) } guard !assignments.isEmpty, diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift index 7c1b7420ad..1ff5996b2d 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift @@ -11,9 +11,6 @@ import TableProTeradataCore /// Teradata literals and the INSERT a new grid row becomes. The driver sends statements as text, so every value is /// written as a literal here rather than bound. enum TeradataRowEditSQL { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func literal(_ value: PluginCellValue) -> String { switch value { case .null: @@ -29,9 +26,10 @@ enum TeradataRowEditSQL { target: String, columns: [String], values: [PluginCellValue], - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) -> String { - let written = zip(columns, values).enumerated().filter { $0.element.1 != defaultMarker } + let written = zip(columns, values).enumerated().filter { !defaultColumns.contains($0.offset) } guard !written.isEmpty else { return "INSERT INTO \(target) DEFAULT VALUES" } let columnList = written.map { TeradataSchemaQueries.quoteIdentifier($0.element.0) }.joined(separator: ", ") let valueList = written.map { cell in @@ -40,9 +38,9 @@ enum TeradataRowEditSQL { return "INSERT INTO \(target) (\(columnList)) VALUES (\(valueList))" } - static func assignment(column: String, value: PluginCellValue, isExpression: Bool) -> String { + static func assignment(column: String, value: PluginCellValue, isExpression: Bool, isDefault: Bool = false) -> String { let name = TeradataSchemaQueries.quoteIdentifier(column) - guard value != defaultMarker else { return "\(name) = DEFAULT" } + guard !isDefault else { return "\(name) = DEFAULT" } return "\(name) = \(Self.value(value, isExpression: isExpression))" } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift index 1cdc981634..86711e30c1 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift @@ -27,7 +27,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText] + [ + .multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText, + .columnDefaultMarks, + ] } func cacheColumnTypes(_ types: [String: String], key: String) { @@ -129,8 +132,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } } } @@ -139,8 +144,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeParameterized(query: query, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeParameterized(query: query, parameters: parameters) + } } } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift index 0a4da883cf..e2cf65dcc6 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift @@ -9,10 +9,7 @@ import TableProTrinoCore /// The column values a grid change writes, in the form `TrinoRowEditSQL` renders. enum TrinoRowEditValues { - /// What the grid stages for a column the user leaves to the server's default. Trino has no `DEFAULT` keyword in - /// `VALUES`, so the column stays out of the insert and the connector fills it. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - + // Trino has no `DEFAULT` keyword in `VALUES`, so a defaulted column stays out of the insert and the connector fills it. static func insertValues( _ change: PluginRowChange, columns: [String], @@ -21,11 +18,11 @@ enum TrinoRowEditValues { ) -> [TrinoColumnValue] { if let rowData = insertedRowData[change.rowIndex] { return columns.enumerated().compactMap { index, column in - guard index < rowData.count, rowData[index] != defaultMarker else { return nil } + guard index < rowData.count, !change.isColumnDefault(index) else { return nil } return TrinoColumnValue(name: column, value: trinoValue(rowData[index]), typeName: typeName(column)) } } - return change.cellChanges.filter { $0.newValue != defaultMarker }.map { + return change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) }.map { TrinoColumnValue(name: $0.columnName, value: trinoValue($0.newValue), typeName: typeName($0.columnName)) } } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift index e00f521323..4ed6c0f139 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift @@ -28,7 +28,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen var supportsTransactions: Bool { false } /// `.userManagement` is what enables Database > Users & Roles, which lists Typesense API keys. - var capabilities: PluginCapabilities { [.cancelQuery, .userManagement] } + var capabilities: PluginCapabilities { [.cancelQuery, .userManagement, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -237,6 +237,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let fields = lock.withLock { _schemaCache[table] }?.fieldsByName ?? [:] let generator = TypesenseStatementGenerator(collection: table, columns: columns, fields: fields) return try generator.generateRowWrites( diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 9c9c7f27ba..f9256485cd 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -81,7 +81,7 @@ extension QueryExecutionCoordinator { sql: String, index: Int, baseQuery: String?, - baseQueryParameterValues: [String?]? = nil, + baseQueryParameterValues: [TypedParameterValue?]? = nil, namedParameterStatement: NamedParameterStatement? = nil, tabId: UUID, anchor: StatementAnchor? = nil, diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index c0bcf998c7..6143d67473 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -286,7 +286,10 @@ extension QueryExecutionCoordinator { ) await MainActor.run { [weak self] in guard let self else { return } - guard parent.tabExecution.settle(claim) else { return } + guard parent.tabExecution.settle(claim) else { + parent.reportStopRollback(error, tabId: tabId) + return + } parent.tabManager.mutate(tabId: tabId) { tab in tab.pagination.isLoadingMore = false } @@ -295,6 +298,7 @@ extension QueryExecutionCoordinator { parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + parent.reportStopRollback(error, tabId: tabId) return } handleQueryExecutionError( @@ -681,7 +685,7 @@ extension QueryExecutionCoordinator { sql: statement.originalSQL, index: firstNumber + index, baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map { $0 as? String }, + baseQueryParameterValues: statement.parameterValues?.map(TypedParameterValue.kept), namedParameterStatement: statement.parameterValues == nil ? nil : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), @@ -790,7 +794,7 @@ extension QueryExecutionCoordinator { prepared: prepared ) - let parameterValues = nativeParameters.map { $0 as? String } + let parameterValues = nativeParameters.map(TypedParameterValue.kept) let statement = originalSQL.map { NamedParameterStatement(sql: $0, parameters: originalParameters) } parent.tabManager.mutate(tabId: tabId) { $0.pagination.baseQueryParameterValues = parameterValues diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 19517845a3..fb8d7ce52a 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -328,23 +328,30 @@ struct RowChangeStatementFactory { return marked } - // A driver reads its own spelling for a default, not the mark, so a default reaches it spelled that way and unmarked. + // A driver built before `PluginRowChange.defaultColumns` reads its own spelling for a default, so a default reaches + // every driver spelled that way, with the mark beside it for one that reads the mark. private static func spellingDefaultsForDrivers( _ changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] - ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]]) { + ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], defaults: [RowID: Set]) { var rows = insertedRowData + var defaults: [RowID: Set] = [:] let spelledChanges = changes.map { change -> RowChange in var spelled = change if change.type == .insert, var values = rows[change.rowID] { for index in change.expressionColumns where values.indices.contains(index) && values[index] == .columnDefault { values[index] = .pluginDefaultMarker spelled.expressionColumns.remove(index) + defaults[change.rowID, default: []].insert(index) } rows[change.rowID] = values } let defaulted = change.cellChanges.filter { $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) } guard !defaulted.isEmpty else { return spelled } + // A new row's stored values are what a driver writes, so only they mark its defaults. + if !(change.type == .insert && insertedRowData[change.rowID] != nil) { + defaults[change.rowID, default: []].formUnion(defaulted.map(\.columnIndex)) + } spelled.cellChanges = change.cellChanges.map { cell in guard cell.newValue.isColumnDefault(isExpression: cell.newIsExpression) else { return cell } return CellChange( @@ -362,7 +369,7 @@ struct RowChangeStatementFactory { } return spelled } - return (spelledChanges, rows) + return (spelledChanges, rows, defaults) } private func rowWriteContext(for keyed: PluginKeyedChanges) -> PluginRowWriteContext { @@ -426,7 +433,9 @@ struct RowChangeStatementFactory { }, context: restoreContext(rowCount: rows.count) ) { - try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) + } try refuseUnfilteredWrites(restored.map { $0.statement }) return RestoreStatements(statements: restored.map { ParameterizedStatement(sql: $0.statement, parameters: $0.parameters.map(\.asAny)) @@ -535,7 +544,8 @@ struct RowChangeStatementFactory { changes: spelled.changes, insertedRowData: spelled.insertedRowData, deletedRowIDs: deletedRowIDs, - insertedRowIDs: insertedRowIDs + insertedRowIDs: insertedRowIDs, + defaultColumns: spelled.defaults ) let writes: [PluginRowWrite] do { @@ -566,7 +576,9 @@ struct RowChangeStatementFactory { throw DataWriteError.changeRefused(table: tableName, kind: nil, reason: error.localizedDescription) } - try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) + } try refuseUnfilteredWrites(writes.map(\.statement)) let unwritten = RowWriteCoverage.unwrittenChanges( changes, @@ -590,8 +602,8 @@ struct RowChangeStatementFactory { } } - // A driver may read the text `__DEFAULT__` as the column default, and none can say whether it does until PluginKit - // carries the mark beside the value, so the text is never handed to one as a value. + // A driver that does not read `PluginRowChange.defaultColumns` may read the text `__DEFAULT__` as the column default, + // so the text is never handed to one as a value. private func refuseTextDriversReadAsDefault( in changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] @@ -709,7 +721,8 @@ struct PluginKeyedChanges { changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], deletedRowIDs: Set, - insertedRowIDs: Set + insertedRowIDs: Set, + defaultColumns: [RowID: Set] = [:] ) { let ordered = changes.sorted { $0.sequence < $1.sequence } var keys: [RowID: Int] = [:] @@ -720,7 +733,7 @@ struct PluginKeyedChanges { } self.rowIDs = rowIDs self.changes = ordered.compactMap { change in - keys[change.rowID].map { PluginRowChange(change, key: $0) } + keys[change.rowID].map { PluginRowChange(change, key: $0, defaultColumns: defaultColumns[change.rowID]) } } self.insertedRowData = Dictionary( uniqueKeysWithValues: insertedRowData.compactMap { rowID, values in @@ -773,7 +786,7 @@ private extension SQLUnfilteredWrite { } private extension PluginRowChange { - init(_ change: RowChange, key: Int) { + init(_ change: RowChange, key: Int, defaultColumns: Set?) { self.init( rowIndex: key, type: { @@ -790,6 +803,7 @@ private extension PluginRowChange { ) absentColumns = Self.absentColumns(after: change) expressionColumns = Self.expressionColumns(of: change) + self.defaultColumns = change.type == .delete || defaultColumns?.isEmpty != false ? nil : defaultColumns } static func expressionColumns(of change: RowChange) -> Set? { diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 587b6fe793..4b4ec3ab6d 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -106,6 +106,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor static func cellValue(for parameter: Any?) -> PluginCellValue { guard let parameter else { return .null } + if let typed = parameter as? TypedParameterValue { return .text(typed.text) } if let data = parameter as? Data { return .bytes(data) } if let f = parameter as? Float { guard f.isFinite else { return .null } @@ -1439,7 +1440,7 @@ extension PluginDriverAdapter { let pluginResult = try await pluginDriver.executeParameterized( query: query, parameters: parameters.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } @@ -1456,14 +1457,15 @@ extension PluginDriverAdapter { query: query, rowCap: rowCap, parameters: parameters?.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } - fileprivate func statementContext(_ query: String, approvedAs caller: PluginStatementContext) -> PluginStatementContext { + fileprivate func statementContext(_ query: String, approvedAs caller: PluginStatementContext, parameters: [Any?]? = nil) -> PluginStatementContext { var context = PluginStatementContext.statement(query, databaseType: connection.type, approvedAs: caller) context.lexicalFeatures = sessionLexicalFeatures + context.parameterKinds = TypedParameterValue.kinds(of: parameters) ?? context.parameterKinds return context } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift index dab2c5e5b4..05251416e1 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift @@ -55,7 +55,9 @@ enum SQLParameterExtractor { } if let param = paramLookup[name] { - values.append(param.isNull ? nil : param.value) + values.append( + param.isNull ? nil : TypedParameterValue(text: param.value, kind: param.type.pluginKind(for: param.value)) + ) } else { values.append(nil) } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index 3c763effc6..acadf5177a 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -14,8 +14,8 @@ import TableProPluginKit internal struct SQLRowToStatementConverter { /// One row of the grid, as it is shown and as the server has it. /// - /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid spells that as the - /// text `__DEFAULT__`, which a stored value can also be, so only a cell named here is read as the marker. + /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid stages that as the + /// keyword DEFAULT marked as SQL, and a stored value can read the same, so only a cell named here is the default. /// `expressionColumns` names the cells staged as SQL, which go out unquoted; text anywhere else is a literal. internal struct SourceRow { internal let values: [PluginCellValue] diff --git a/apple/TablePro/Models/Query/QueryParameter.swift b/apple/TablePro/Models/Query/QueryParameter.swift index d7a7cc4db8..56550eab6a 100644 --- a/apple/TablePro/Models/Query/QueryParameter.swift +++ b/apple/TablePro/Models/Query/QueryParameter.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProPluginKit enum QueryParameterType: String, Codable, CaseIterable, Sendable { case string @@ -28,3 +29,56 @@ struct QueryParameter: Identifiable, Codable, Equatable, Hashable, Sendable { self.isNull = isNull } } + +extension QueryParameterType { + // String is the panel's default, so a plain number in it still goes as a number (`LIMIT :n`, `id = :id`). Anything + // else is text, a code like `1E5` included; Decimal sends that as a number. + func pluginKind(for value: String) -> PluginColumnKind? { + switch self { + case .string: return Self.isPlainNumber(value) ? nil : .text + case .integer: return .integer + case .decimal: return .decimal + case .boolean: return .boolean + case .date: return .other + } + } + + // Digits with an optional sign and an optional fraction: `42`, `-3.5`, `.5`. + private static func isPlainNumber(_ value: String) -> Bool { + var scalars = value.unicodeScalars[...] + if scalars.first == "-" || scalars.first == "+" { scalars = scalars.dropFirst() } + var sawDot = false + var digitsAfterDot = 0 + var digits = 0 + for scalar in scalars { + if ("0"..."9").contains(scalar) { + digits += 1 + if sawDot { digitsAfterDot += 1 } + } else if scalar == ".", !sawDot { + sawDot = true + } else { + return false + } + } + return digits > 0 && (!sawDot || digitsAfterDot > 0) + } +} + +// A panel value with the type the user gave it, so a driver that writes values into the statement writes it as that. +struct TypedParameterValue: Equatable, Sendable { + let text: String + let kind: PluginColumnKind? + + // A run's value as the rerun state keeps it; plain text from a caller that gave no type stays text with no kind. + static func kept(_ value: Any?) -> TypedParameterValue? { + if let typed = value as? TypedParameterValue { return typed } + return (value as? String).map { TypedParameterValue(text: $0, kind: nil) } + } + + // Nil when no value carries a type, so a driver reads the values as it did before types were passed. + static func kinds(of parameters: [Any?]?) -> [PluginColumnKind?]? { + guard let parameters else { return nil } + let kinds = parameters.map { ($0 as? TypedParameterValue)?.kind ?? nil } + return kinds.contains { $0 != nil } ? kinds : nil + } +} diff --git a/apple/TablePro/Models/Query/QueryTabState.swift b/apple/TablePro/Models/Query/QueryTabState.swift index 796c43ee88..48b832dd2e 100644 --- a/apple/TablePro/Models/Query/QueryTabState.swift +++ b/apple/TablePro/Models/Query/QueryTabState.swift @@ -288,10 +288,10 @@ struct PaginationState: Equatable { var hasMoreRows: Bool = false var isLoadingMore: Bool = false var baseQueryForMore: String? - var baseQueryParameterValues: [String?]? + var baseQueryParameterValues: [TypedParameterValue?]? // Set together: stale bindings sent with a placeholder-free query made Fetch All fail on the server. - mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [String?]?) { + mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [TypedParameterValue?]?) { baseQueryForMore = sql baseQueryParameterValues = parameterValues } diff --git a/apple/TablePro/Models/Query/ResultSet.swift b/apple/TablePro/Models/Query/ResultSet.swift index 96319f8afa..44e62e8543 100644 --- a/apple/TablePro/Models/Query/ResultSet.swift +++ b/apple/TablePro/Models/Query/ResultSet.swift @@ -39,7 +39,7 @@ final class ResultSet: ObservableObject, Identifiable { @Published var isPinned: Bool = false @Published var isTruncated: Bool = false @Published var baseQuery: String? - @Published var baseQueryParameterValues: [String?]? + @Published var baseQueryParameterValues: [TypedParameterValue?]? /// The statement a parameterized run produced these rows from, as written, with the values it bound. `baseQuery` /// and `baseQueryParameterValues` are the same run in the driver's positional form, which Fetch All sends as it diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index ebbd35aef7..570eb50fdb 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -44054,6 +44054,10 @@ } } }, + "column %@ first appears in a later chunk" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Column %@ has a type, default, generation expression or check that is not one SQL expression." : { "extractionState" : "manual", "generatesSymbol" : false @@ -194590,6 +194594,10 @@ } } }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The statement's placeholders do not match the %d values given." : { "extractionState" : "manual", "generatesSymbol" : false @@ -198401,6 +198409,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift index 48861aa5be..2e1a8f6b71 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift @@ -20,6 +20,13 @@ extension MainContentCoordinator { ) } + // Stop ends the run before the driver answers, so a rollback the server did for it arrives on a settled claim. It is + // shown on the tab unless the tab already runs something new. + func reportStopRollback(_ error: Error, tabId: UUID) { + guard error is PluginStopRolledBackTransaction, !tabExecution.isExecuting(tabId) else { return } + tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = error.localizedDescription } + } + /// A table tab's SELECT is the app's own, so it may follow a database switch it waited through /// onto a pooled connection. The choice belongs to the tab and never to the statement: an editor /// SELECT can read a temp table or sit inside the user's open transaction, and moving it to @@ -57,6 +64,7 @@ extension MainContentCoordinator { serverOutput: PluginServerOutput = .none ) { guard tabExecution.settle(claim) else { + reportStopRollback(error, tabId: tabId) traceStaleResultDropped(traceToken) return } @@ -73,6 +81,7 @@ extension MainContentCoordinator { reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + reportStopRollback(error, tabId: tabId) return } if isAutoLoad, services.databaseManager.driver(for: connectionId)?.status != .connected { diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift index 0389ae815c..ab6c89d52a 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift @@ -79,8 +79,8 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, columns: [String], values: [(column: String, value: String?)] ) throws -> [PluginRowWrite] { - // The plugin reads this text as "leave the column out", so storing it as a value cannot be expressed. - if values.contains(where: { $0.value == Self.defaultSentinel }) { + // A plugin that does not read the default mark reads this text as "leave the column out", so it cannot store it. + if !plugin.capabilities.contains(.columnDefaultMarks), values.contains(where: { $0.value == Self.defaultSentinel }) { throw DatabaseError(message: String(format: String( localized: "The text %@ cannot be inserted: this database reads it as the column's default." ), Self.defaultSentinel)) @@ -90,7 +90,9 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, guard let value = given[column] else { return .text(Self.defaultSentinel) } return PluginCellValue(optionalText: value) } - let rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let defaulted = Set(columns.indices.filter { given[columns[$0]] == nil }) + rowChange.defaultColumns = defaulted.isEmpty ? nil : defaulted return try generate( table: table, schema: schema, columns: columns, changes: [rowChange], inserted: [0: row], context: PluginRowWriteContext() @@ -158,7 +160,7 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, DatabaseError(message: String(localized: "This database cannot write this change from iPhone or iPad.")) } - // The Mac's grid marks a cell left at the column default with this text, and the drivers leave such a column out. + // What a column left at its default holds, for a plugin built before `PluginRowChange.defaultColumns`. static let defaultSentinel = "__DEFAULT__" private func generate( diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 6c3e6f49ca..8a20343c3c 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -26443,6 +26443,10 @@ }, "The statement before it stays applied." : { + }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The table \"%@\" and all its data will be permanently deleted." : { "localizations" : { @@ -27177,6 +27181,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift index 910302a3d6..1bbe8b04d4 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift @@ -231,6 +231,34 @@ struct PluginBackedDriverTests { ]) } + @Test("A plugin that reads the default mark gets every left-out column marked") + func insertMarksOmittedColumnsForAMarkingPlugin() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "name", "note"], values: [(column: "name", value: "x")] + ) + + #expect(plugin.generatedChanges.first?.defaultColumns == [0, 2]) + } + + @Test("A plugin that reads the default mark stores the marker's text as a value") + func markingPluginStoresTheMarkerText() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "note"], + values: [(column: "id", value: "1"), (column: "note", value: PluginRowWritingDriver.defaultSentinel)] + ) + + #expect(plugin.generatedInserts[0] == ["1", .text(PluginRowWritingDriver.defaultSentinel)]) + #expect(plugin.generatedChanges.first?.defaultColumns == nil) + } + @Test("A value spelled as the default marker is refused, never stored as the default") func insertRefusesTheDefaultMarker() { let plugin = StubPluginDriver() diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift index 376ba32e9c..e5ede52867 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift @@ -12,8 +12,11 @@ final class StubPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var checkRows: [Int] = [] var onFetchTables: (@Sendable () async -> Void)? var currentDb = "" + var declaredCapabilities: PluginCapabilities = [] private(set) var databasesAtExecute: [String] = [] + var capabilities: PluginCapabilities { declaredCapabilities } + private(set) var boundedQueries: [(query: String, rowCap: Int)] = [] private(set) var plainQueries: [String] = [] private(set) var parameterizedQueries: [(query: String, parameters: [PluginCellValue])] = [] diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift index 0109aeb1ec..376fdb75e2 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift @@ -37,6 +37,12 @@ private final class ExpressionRecordingDriver: PluginDriverStub, @unchecked Send } private final class RestoringDriver: PluginDriverStub, @unchecked Sendable { + let capabilities: PluginCapabilities + + init(capabilities: PluginCapabilities = []) { + self.capabilities = capabilities + } + func generateIdentityPreservingInsert( table: String, schema: String?, @@ -286,6 +292,66 @@ struct RowChangeStatementFactoryExpressionTests { #expect(refused?.reason.contains("name") == true) } + @Test("A driver that reads the default mark gets the mark beside its old spelling") + func defaultReachesAMarkingDriverWithItsMark() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.defaultColumns == [1]) + #expect(driver.changes.first?.expressionColumns == nil) + } + + @Test("A driver that reads the default mark saves the text __DEFAULT__ as text") + func textAMarkingDriverReadsIsSaved() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + #expect(driver.changes.first?.defaultColumns == nil) + } + + @Test("A new row's default cells are marked for the driver, and its text cells are not") + func newRowDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let rowID = RowID.inserted(UUID()) + let change = RowChange(rowID: rowID, type: .insert, expressionColumns: [0]) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [change], + insertedRowData: [rowID: [.columnDefault, "__DEFAULT__", "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(driver.changes.first?.defaultColumns == [0]) + } + + @Test("A new row staged only as cell edits marks its default cells by column") + func newRowCellEditDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let change = RowChange( + rowID: .inserted(UUID()), + type: .insert, + cellChanges: [ + CellChange(columnIndex: 1, columnName: "name", oldValue: .null, newValue: "__DEFAULT__"), + CellChange( + columnIndex: 2, columnName: "stamped_at", oldValue: .null, newValue: .columnDefault, + newIsExpression: true + ), + ] + ) + _ = try factory(databaseType: .mssql, driver: driver).statements(for: [change]) + #expect(driver.changes.first?.defaultColumns == [2]) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + } + + @Test("A deleted row holding that text is put back by a driver that reads the default mark") + func restoredTextAMarkingDriverReadsIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver(capabilities: .columnDefaultMarks)) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "__DEFAULT__", "2026-01-01"]) + } + @Test("A deleted row a driver puts back with no such text is restored") func restoredRowWithoutThatTextIsWritten() throws { let restored = try factory(databaseType: .dameng, driver: RestoringDriver()) diff --git a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift index 775152ce39..8844e795ba 100644 --- a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift @@ -320,3 +320,42 @@ struct UnenforcedStatementPolicyTests { #expect(adapter(ContextRecordingDriver(capabilities: .readOnlyStatements)).runsReadOnlyStatements) } } + +struct PluginDriverAdapterParameterKindTests { + private func adapter(_ driver: ContextRecordingDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .postgresql), pluginDriver: driver) + } + + @Test("A panel value reaches the driver with the type the user gave it") + func panelTypesReachTheDriver() async throws { + let driver = ContextRecordingDriver() + let values: [Any?] = [ + TypedParameterValue(text: "00123", kind: .text), nil, TypedParameterValue(text: "7", kind: .integer), + ] + + _ = try await adapter(driver).executeUserQuery( + query: "SELECT $1, $2, $3", rowCap: nil, parameters: values, context: PluginStatementContext() + ) + _ = try await adapter(driver).executeParameterized( + query: "SELECT $1, $2, $3", parameters: values, context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [[.text, nil, .integer], [.text, nil, .integer]]) + } + + @Test("Values with no type, such as a grid save's, carry none") + func untypedValuesCarryNoKinds() async throws { + let driver = ContextRecordingDriver() + + _ = try await adapter(driver).executeParameterized( + query: "UPDATE t SET a = $1", parameters: ["42"], context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [nil]) + } + + @Test("A typed value is written to the driver as its text") + func typedValueIsItsText() { + #expect(PluginDriverAdapter.cellValue(for: TypedParameterValue(text: "1E5", kind: .text)) == .text("1E5")) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift index 17f6c3c005..12629b8c26 100644 --- a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift @@ -132,8 +132,16 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE id = ? AND name = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "Alice") + #expect((result.values[0] as? TypedParameterValue) == TypedParameterValue(text: "42", kind: .integer)) + #expect((result.values[1] as? TypedParameterValue) == TypedParameterValue(text: "Alice", kind: .text)) + } + + @Test("A String parameter holding a plain number keeps the reading by shape; any other text is text", arguments: [ + ("42", PluginColumnKind?.none), ("-3.5", nil), ("00123", nil), ("1E5", .text), ("12e3", .text), ("1.", .text), + (".5", nil), ("abc", .text), ("", .text), ("-", .text), + ]) + func stringKindFollowsTheValue(value: String, kind: PluginColumnKind?) { + #expect(QueryParameterType.string.pluginKind(for: value) == kind) } @Test @@ -161,8 +169,8 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE ? = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "42") + #expect((result.values[0] as? TypedParameterValue)?.text == "42") + #expect((result.values[1] as? TypedParameterValue)?.text == "42") } @Test diff --git a/apple/TableProTests/Models/PaginationStateTests.swift b/apple/TableProTests/Models/PaginationStateTests.swift index 0235101ebe..35e00686ec 100644 --- a/apple/TableProTests/Models/PaginationStateTests.swift +++ b/apple/TableProTests/Models/PaginationStateTests.swift @@ -331,7 +331,7 @@ struct PaginationStateTests { func resetLoadMoreClearsTheQueryAndItsValues() { var state = PaginationState() state.hasMoreRows = true - state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: ["1"]) + state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: [TypedParameterValue(text: "1", kind: .text)]) state.resetLoadMore() @@ -404,7 +404,7 @@ struct PaginationStateTests { @Test("Setting the base query clears the previous run's bindings") func setBaseQueryForMoreClearsStaleBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["42"]) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [TypedParameterValue(text: "42", kind: .text)]) state.setBaseQueryForMore("SELECT * FROM events", parameterValues: nil) @@ -416,10 +416,11 @@ struct PaginationStateTests { func setBaseQueryForMoreCarriesBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["7"]) + let bound = TypedParameterValue(text: "7", kind: .integer) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [bound]) #expect(state.baseQueryForMore == "SELECT * FROM orders WHERE id = $1") - #expect(state.baseQueryParameterValues == ["7"]) + #expect(state.baseQueryParameterValues == [bound]) } @Test("Busy covers every kind of pending work") diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift index 47cd3e727d..14cee2efb9 100644 --- a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift @@ -528,7 +528,7 @@ struct MainContentCoordinatorSortTests { let idx = try #require(tabManager.tabs.firstIndex(where: { $0.id == tabId })) let result = ResultSet(label: "audit", tableRows: rows) result.baseQuery = "SELECT msg FROM audit WHERE msg = ?" - result.baseQueryParameterValues = ["hello"] + result.baseQueryParameterValues = [TypedParameterValue(text: "hello", kind: .text)] tabManager.mutate(at: idx) { $0.display.replaceUnpinnedResults(with: [result]) } coordinator.setActiveTableRows(rows, for: tabId) @@ -894,7 +894,7 @@ private final class StatementRecordingDriver: DatabaseDriver, @unchecked Sendabl } private func record(_ query: String, parameters: [Any?]?) -> QueryResult { - let bound = parameters.map { values in values.map { $0 as? String } } + let bound = parameters.map { values in values.map { ($0 as? TypedParameterValue)?.text ?? $0 as? String } } lock.withLock { statements.append(SentStatement(sql: query, parameters: bound)) } guard query.uppercased().hasPrefix("SELECT") else { return QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 1, executionTime: 0, error: nil) diff --git a/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift new file mode 100644 index 0000000000..b75f6881d9 --- /dev/null +++ b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift @@ -0,0 +1,40 @@ +// +// StopRollbackNoticeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct StopRollbackNoticeTests { + private func fixture() -> (CoordinatorFixture, UUID) { + let fixture = CoordinatorFixture(connection: TestFixtures.makeConnection(type: .postgresql)) + let tab = fixture.addTab(QueryTab(title: "Q1", query: "UPDATE t SET a = 1", tabType: .query)) + return (fixture, tab.id) + } + + private func errorMessage(_ fixture: CoordinatorFixture, _ tabId: UUID) -> String? { + fixture.tabManager.tabs.first { $0.id == tabId }?.execution.errorMessage + } + + @Test("A rollback the server did for a Stop is shown on the stopped tab") + func rollbackIsShown() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(PluginStopRolledBackTransaction(code: 3_110), tabId: tabId) + + #expect(errorMessage(fixture, tabId)?.contains("rolled back the transaction") == true) + } + + @Test("A plain cancel leaves the tab as Stop left it") + func plainCancelShowsNothing() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(CancellationError(), tabId: tabId) + + #expect(errorMessage(fixture, tabId) == nil) + } +} diff --git a/apple/changelog.d/4190.fixed.1.md b/apple/changelog.d/4190.fixed.1.md new file mode 100644 index 0000000000..ecc80d29b8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.1.md @@ -0,0 +1 @@ +Saving the text __DEFAULT__ in a SQL Server, Oracle, MongoDB or other driver-written table was refused. diff --git a/apple/changelog.d/4190.fixed.2.md b/apple/changelog.d/4190.fixed.2.md new file mode 100644 index 0000000000..74b5117d28 --- /dev/null +++ b/apple/changelog.d/4190.fixed.2.md @@ -0,0 +1 @@ +Exporting a large Teradata, InfluxDB, SAP HANA or Memcached table loaded every row into memory first. diff --git a/apple/changelog.d/4190.fixed.3.md b/apple/changelog.d/4190.fixed.3.md new file mode 100644 index 0000000000..f49fe3cc89 --- /dev/null +++ b/apple/changelog.d/4190.fixed.3.md @@ -0,0 +1 @@ +Stopping a Teradata query in ANSI mode did not say that Teradata rolled back the open transaction. diff --git a/apple/changelog.d/4190.fixed.4.md b/apple/changelog.d/4190.fixed.4.md new file mode 100644 index 0000000000..ce44ae8112 --- /dev/null +++ b/apple/changelog.d/4190.fixed.4.md @@ -0,0 +1 @@ +Oracle TIMESTAMP WITH TIME ZONE values stored with a region name showed a placeholder instead of the time. diff --git a/apple/changelog.d/4190.fixed.5.md b/apple/changelog.d/4190.fixed.5.md new file mode 100644 index 0000000000..7a22306a09 --- /dev/null +++ b/apple/changelog.d/4190.fixed.5.md @@ -0,0 +1 @@ +A Teradata query with the row limit on read every row of the result. diff --git a/apple/changelog.d/4190.fixed.6.md b/apple/changelog.d/4190.fixed.6.md new file mode 100644 index 0000000000..76236db0a8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.6.md @@ -0,0 +1 @@ +A long SAP HANA export failed when it ran past the query timeout. diff --git a/apple/changelog.d/4190.fixed.7.md b/apple/changelog.d/4190.fixed.7.md new file mode 100644 index 0000000000..b130b47acb --- /dev/null +++ b/apple/changelog.d/4190.fixed.7.md @@ -0,0 +1 @@ +Duplicating an Elasticsearch document saved the copy with the id __DEFAULT__. diff --git a/apple/changelog.d/4190.fixed.8.md b/apple/changelog.d/4190.fixed.8.md new file mode 100644 index 0000000000..29f5f55088 --- /dev/null +++ b/apple/changelog.d/4190.fixed.8.md @@ -0,0 +1 @@ +Stopping a Memcached export left the connection closed until a reconnect. diff --git a/apple/changelog.d/4190.fixed.md b/apple/changelog.d/4190.fixed.md new file mode 100644 index 0000000000..4d55e2759e --- /dev/null +++ b/apple/changelog.d/4190.fixed.md @@ -0,0 +1 @@ +Integer, Decimal and Boolean query parameters, and codes such as 1E5, were sent to Trino by how the value looked. diff --git a/apple/changelog.d/4190.security.md b/apple/changelog.d/4190.security.md new file mode 100644 index 0000000000..900541eb6a --- /dev/null +++ b/apple/changelog.d/4190.security.md @@ -0,0 +1 @@ +Browsing a Memcached cache with a key holding a line break could run a second command on the server. diff --git a/apple/scripts/probes/check-oracle-time-zone-regions.sh b/apple/scripts/probes/check-oracle-time-zone-regions.sh new file mode 100755 index 0000000000..bd6a1beab8 --- /dev/null +++ b/apple/scripts/probes/check-oracle-time-zone-regions.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# +# Check that a TIMESTAMP WITH TIME ZONE stored with a region name reads as its wall clock and Oracle's name, and that +# the grid's write-back masks parse that text again. +# +# Such a value carries an id into the server's time zone file. OracleTimeZoneRegions reads the ids once per connection +# from SYS.V_$TIMEZONE_NAMES and assumes a layout no Oracle document states: 0x80 in byte 11 marks a region, the id is +# 13 bits over bytes 11 and 12 with the low two bits of byte 12 unused, and bytes 1 to 7 are UTC. This reads the wire +# bytes through oracle-nio and the cell text through the same OracleCoreConnection the app uses, and checks: +# 1. whether the account can read the view; ORA-00942 is a normal answer, and the cells then read as UTC +# 2. the map query's rows, for AT TIME ZONE and FROM_TZ: every row a region, no id repeated, the low bits clear +# 3. which names Foundation's TimeZone does not know +# 4. stored region values: wire bytes against SYS_EXTRACT_UTC and DUMP, cell text against TO_CHAR and EXTRACT +# 5. a session zone set to a region: CURRENT_TIMESTAMP and friends read with it, and the map outlives the ALTER +# 6. the masks in OracleRowWriter.swift parse the region, offset and zone-less spellings, into TSTZ and LTZ +# +# Usage: +# scripts/probes/check-oracle-time-zone-regions.sh [host] [port] [service] [user] [password] +# +# Defaults suit a throwaway Oracle Free container: +# docker run -d --name oracle-probe -p 1521:1521 -e ORACLE_PASSWORD=probe_pw \ +# -e APP_USER=probe -e APP_USER_PASSWORD=probe_pw gvenzl/oracle-free:23-slim-faststart +# +# Hand-run only, never in CI. The user needs CREATE TABLE. Every object it creates is prefixed TP_TZR_ and dropped +# again. Exits non-zero on a disagreement, 3 when a prerequisite is missing. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-1521}" +SERVICE="${3:-FREEPDB1}" +USER_NAME="${4:-probe}" +PASSWORD="${5:-probe_pw}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [ -z "${DEVELOPER_DIR:-}" ]; then + for candidate in /Applications/Xcode-beta.app /Applications/Xcode.app; do + if [ -d "$candidate/Contents/Developer" ]; then + export DEVELOPER_DIR="$candidate/Contents/Developer" + break + fi + done +fi +command -v swift > /dev/null || { + echo "swift not found" >&2 + exit 3 +} + +MASK="$(sed -n 's/.*static let timestampWithTimeZoneMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +OFFSET_MASK="$(sed -n 's/.*static let timestampWithOffsetMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +NIO_REVISION="$(sed -n 's/.*revision: "\([0-9a-f]*\)".*/\1/p' "$ROOT/Packages/TableProOracle/Package.swift")" +if [ -z "$MASK" ] || [ -z "$OFFSET_MASK" ] || [ -z "$NIO_REVISION" ]; then + echo "cannot read the writer's mask or the oracle-nio pin" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/Probe" + +cat > "$WORK/Package.swift" < "$WORK/Sources/Probe/main.swift" <<'EOF' +import Foundation +import NIOCore +import OracleNIO +import TableProOracleCore + +let arguments = CommandLine.arguments +let host = arguments[1] +let port = Int(arguments[2]) ?? 1_521 +let service = arguments[3] +let user = arguments[4] +let password = arguments[5] +let regionMask = arguments[6] +let offsetMask = arguments[7] +let view = "SYS.V_$TIMEZONE_NAMES" + +let connection = OracleCoreConnection(options: OracleConnectionOptions( + host: host, port: port, user: user, password: password, identifierMode: .service, serviceName: service +)) +do { + try await connection.connect() +} catch { + print("cannot connect: \(error)") + exit(3) +} + +func dialRaw() async -> OracleNIO.OracleConnection { + do { + return try await OracleNIO.OracleConnection.connect( + configuration: OracleNIO.OracleConnection.Configuration( + host: host, port: port, service: .serviceName(service), username: user, password: password + ), + id: 1 + ) + } catch { + print("cannot connect through oracle-nio: \(error)") + exit(3) + } +} + +let raw = await dialRaw() + +var disagreements = 0 + +@MainActor +func check(_ holds: Bool, _ claim: String) { + print("\(holds ? "ok " : "FAIL") \(claim)") + if !holds { disagreements += 1 } +} + +func info(_ line: String) { + print("info \(line)") +} + +func attempt(_ sql: String) async -> (result: OracleRawResult?, failure: String?) { + do { + return (try await connection.executeQuery(sql), nil) + } catch { + return (nil, String(describing: error)) + } +} + +@MainActor +func result(_ sql: String) async -> OracleRawResult? { + let answer = await attempt(sql) + if let failure = answer.failure { + print("FAIL \(sql.prefix(80)): \(failure)") + disagreements += 1 + } + return answer.result +} + +func scalar(_ sql: String) async -> String? { + await attempt(sql).result?.rows.first?.first?.stringValue +} + +func run(_ sql: String) async { + _ = await attempt(sql) +} + +// The cells as oracle-nio hands them over, before the core decodes anything. +func rawRows(_ sql: String) async throws -> [[OracleCell]] { + let stream = try await raw.execute( + OracleStatement(unsafeSQL: sql), options: StatementOptions(prefetchRows: 1_000, arraySize: 1_000) + ) + var rows: [[OracleCell]] = [] + for try await row in stream { + rows.append(Array(row)) + } + return rows +} + +func bytes(of cell: OracleCell) -> [UInt8] { + cell.bytes.map { Array($0.readableBytesView) } ?? [] +} + +func hex(_ bytes: [UInt8]) -> String { + bytes.map { String(format: "%02x", $0) }.joined() +} + +func listed(_ items: [String]) -> String { + items.isEmpty ? "" : " (\(items.count): \(items.prefix(5).joined(separator: "; ")))" +} + +// The writer's own choice: a region name after a blank takes the region mask, anything else the offset mask. +func mask(for text: String) -> String { + text.split(separator: " ").last?.first?.isLetter == true ? regionMask : offsetMask +} + +func parsed(_ text: String) -> String { + "TO_TIMESTAMP_TZ('\(text)', '\(mask(for: text))')" +} + +guard let release = connection.serverRelease else { + print("the login reply carried no server release") + exit(3) +} +print("Oracle \(release.major).\(release.update), writer masks '\(regionMask)' and '\(offsetMask)'") + +print("\n1. Who can read \(view)") +let roles = await result("SELECT ROLE FROM SESSION_ROLES ORDER BY ROLE")?.rows.compactMap { $0.first?.stringValue } ?? [] +info("session roles: \(roles.isEmpty ? "none" : roles.joined(separator: ", "))") +let publicGrants = await scalar(""" + SELECT COUNT(*) FROM ALL_TAB_PRIVS + WHERE TABLE_SCHEMA = 'SYS' AND TABLE_NAME = 'V_$TIMEZONE_NAMES' AND GRANTEE = 'PUBLIC' + """) +info("PUBLIC grants on \(view) in ALL_TAB_PRIVS: \(publicGrants ?? "unreadable")") +let access = await attempt("SELECT COUNT(*) FROM \(view)") +let mapReadable = access.result != nil +if let count = access.result?.rows.first?.first?.stringValue { + check(true, "\(user) reads \(view): \(count) rows") +} else { + let failure = access.failure ?? "" + check(failure.contains("ORA-00942"), "\(user) is refused \(view) with ORA-00942: \(failure.prefix(120))") +} +if let file = await scalar("SELECT FILENAME || ' version ' || VERSION FROM SYS.V_$TIMEZONE_FILE") { + info("time zone file \(file)") +} + +print("\n2. The map query, read raw") +var mapRows: [(name: String, wire: [UInt8])] = [] +var idsByForm: [String: [String: Int]] = [:] +let fromTZQuery = "SELECT TZNAME, FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', TZNAME) FROM " + + "(SELECT DISTINCT TZNAME FROM \(view)) ORDER BY TZNAME" +if mapReadable { + for (form, sql) in [("AT TIME ZONE", OracleTimeZoneRegions.query), ("FROM_TZ", fromTZQuery)] { + let started = ContinuousClock.now + let rows: [[OracleCell]] + do { + rows = try await rawRows(sql) + } catch { + check(false, "\(form): the map query runs: \(error)") + continue + } + info("\(form): \(rows.count) rows in \(ContinuousClock.now - started)") + let pairs = rows.compactMap { cells -> (name: String, wire: [UInt8])? in + guard cells.count == 2, let name = try? cells[0].decode(String.self) else { return nil } + return (name, bytes(of: cells[1])) + } + let unflagged = pairs.filter { OracleTimeZoneRegions.regionID(wire: $0.wire) == nil }.map { "\($0.name) \(hex($0.wire))" } + check(unflagged.isEmpty, "\(form): every row carries a region id\(listed(unflagged))") + var namesByID: [Int: [String]] = [:] + for pair in pairs { + guard let id = OracleTimeZoneRegions.regionID(wire: pair.wire) else { continue } + namesByID[id, default: []].append(pair.name) + } + let shared = namesByID.filter { $0.value.count > 1 }.map { "\($0.key): \($0.value.joined(separator: ", "))" } + check(shared.isEmpty, "\(form): no two names share an id\(listed(shared))") + let lowBits = pairs.filter { $0.wire.count >= 13 && $0.wire[12] & 0x03 != 0 }.map { "\($0.name) \(hex($0.wire))" } + check(lowBits.isEmpty, "\(form): the low two bits of byte 12 are clear\(listed(lowBits))") + var ids: [String: Int] = [:] + for pair in pairs { + ids[pair.name] = OracleTimeZoneRegions.regionID(wire: pair.wire) + } + idsByForm[form] = ids + if form == "AT TIME ZONE" { + mapRows = pairs + let utcFields: [UInt8] = [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01] + let moved = pairs.filter { Array($0.wire.prefix(7)) != utcFields }.map { "\($0.name) \(hex($0.wire))" } + check(moved.isEmpty, "AT TIME ZONE: bytes 1 to 7 of every row are 2000-01-01 00:00:00 UTC\(listed(moved))") + } + } + if idsByForm.count == 2 { + check(idsByForm["AT TIME ZONE"] == idsByForm["FROM_TZ"], "AT TIME ZONE and FROM_TZ give each name the same id") + } + info("OracleTimeZoneRegions keeps \(OracleTimeZoneRegions(rows: mapRows).count) of \(mapRows.count) rows") +} else { + info("skip: \(view) is not readable, so every region value reads as UTC") +} + +// The first name per id, as the map keeps it: the query orders by name. +var nameByID: [Int: String] = [:] +for row in mapRows { + guard let id = OracleTimeZoneRegions.regionID(wire: row.wire), nameByID[id] == nil else { continue } + nameByID[id] = row.name +} + +print("\n3. Names Foundation does not know") +let unknownNames = mapRows.map { $0.name }.filter { TimeZone(identifier: $0) == nil } +info("\(unknownNames.count) of \(mapRows.count) names read as UTC instead") +for name in unknownNames { + info("Foundation does not know \(name)") +} + +print("\n4. Stored region values") +let samples: [(id: Int, literal: String)] = [ + (1, "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), + (2, "TIMESTAMP '2024-07-01 14:00:00 Europe/Paris'"), + (3, "TIMESTAMP '2024-03-10 01:30:00 America/Los_Angeles'"), + (4, "TIMESTAMP '2024-11-03 12:00:00 US/Pacific'"), + (5, "TIMESTAMP '2024-06-15 08:00:00 UTC'"), + (6, "TIMESTAMP '2024-06-15 08:00:00 Etc/GMT+5'"), + (7, "TIMESTAMP '1880-06-01 12:00:00 Europe/Paris'"), + (8, "TIMESTAMP '2024-10-27 02:30:00 Europe/Paris CEST'"), +] +let overlapRow = 8 +await run("DROP TABLE TP_TZR_T PURGE") +if let failure = await attempt("CREATE TABLE TP_TZR_T (ID NUMBER PRIMARY KEY, TZ TIMESTAMP(6) WITH TIME ZONE)").failure { + print("cannot create TP_TZR_T: \(failure)") + exit(3) +} +var literalByID: [Int: String] = [:] +for sample in samples { + if let failure = await attempt("INSERT INTO TP_TZR_T VALUES (\(sample.id), \(sample.literal))").failure { + info("skip \(sample.literal): \(failure.prefix(100))") + } else { + literalByID[sample.id] = sample.literal + } +} +_ = await result("COMMIT") + +var wireByID: [Int: [UInt8]] = [:] +do { + for cells in try await rawRows("SELECT TO_CHAR(ID), TZ, SYS_EXTRACT_UTC(TZ), DUMP(TZ) FROM TP_TZR_T ORDER BY ID") { + guard cells.count == 4, let row = (try? cells[0].decode(String.self)).flatMap({ Int($0) }) else { continue } + let wire = bytes(of: cells[1]) + let utc = bytes(of: cells[2]) + let dump = (try? cells[3].decode(String.self)) ?? "" + let dumped = dump.split(separator: ":", maxSplits: 1).dropFirst().first.map { + $0.split(separator: ",").compactMap { UInt8($0.trimmingCharacters(in: .whitespaces)) } + } ?? [] + let id = OracleTimeZoneRegions.regionID(wire: wire) + let lowBits = wire.count >= 13 ? String(wire[12] & 0x03) : "?" + info("row \(row): \(hex(wire)), region id \(id.map { String($0) } ?? "none"), byte 12 low bits \(lowBits)") + check(id != nil, "row \(row): the stored value carries a region id") + check(Array(wire.prefix(7)) == Array(utc.prefix(7)), "row \(row): wire bytes 1 to 7 are SYS_EXTRACT_UTC's \(hex(Array(utc.prefix(7))))") + check(dump.hasPrefix("Typ=181") && dumped == wire, "row \(row): DUMP of the column matches the wire bytes: \(dump)") + wireByID[row] = wire + } +} catch { + check(false, "the stored values read raw: \(error)") +} + +let textQuery = """ + SELECT TO_CHAR(ID), TZ, TO_CHAR(TZ, 'YYYY-MM-DD HH24:MI:SS.FF6') || ' ' || EXTRACT(TIMEZONE_REGION FROM TZ), + TO_CHAR(SYS_EXTRACT_UTC(TZ), 'YYYY-MM-DD HH24:MI:SS.FF6') || '+00:00', + EXTRACT(TIMEZONE_REGION FROM TZ), TO_CHAR(TZ, 'TZR') + FROM TP_TZR_T ORDER BY ID + """ +var cellTextByID: [Int: String] = [:] +var regionByID: [Int: String] = [:] +for values in await result(textQuery)?.rows ?? [] { + guard values.count == 6, let row = values[0].stringValue.flatMap({ Int($0) }) else { continue } + let cell = values[1].stringValue ?? "NULL" + let server = values[2].stringValue ?? "" + let utcText = values[3].stringValue ?? "" + let region = values[4].stringValue ?? "" + let tzr = values[5].stringValue ?? "" + let mapName = wireByID[row].flatMap { OracleTimeZoneRegions.regionID(wire: $0) }.flatMap { nameByID[$0] } + if let mapName, mapName != region { + check(false, "row \(row): the map names its id \(mapName), the server \(region)") + } + let expected = mapName.map { TimeZone(identifier: $0) != nil } == true ? server : utcText + check(cell == expected, "row \(row) \(literalByID[row] ?? ""): reads \(cell)\(cell == expected ? "" : ", expected \(expected)")") + if tzr != region { + info("row \(row): TO_CHAR TZR spells \(tzr), EXTRACT spells \(region)") + } + cellTextByID[row] = cell + regionByID[row] = region +} + +print("\n5. A session zone set to a region") +_ = await result("ALTER SESSION SET TIME_ZONE = 'Europe/Paris'") +let sessionSuffix = nameByID.values.contains("Europe/Paris") ? " Europe/Paris" : "+00:00" +let regionForms = [ + "CURRENT_TIMESTAMP", + "FROM_TZ(TIMESTAMP '2024-01-02 03:04:05', 'Europe/Paris')", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT TIME ZONE 'Europe/Paris'", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT LOCAL", + "CAST(CAST(TIMESTAMP '2024-01-02 02:04:05 UTC' AS TIMESTAMP WITH LOCAL TIME ZONE) AS TIMESTAMP WITH TIME ZONE)", +] +if let values = await result("SELECT \(regionForms.joined(separator: ", ")) FROM DUAL")?.rows.first { + for (form, value) in zip(regionForms, values) { + let text = value.stringValue ?? "NULL" + check(text.hasSuffix(sessionSuffix), "\(form) reads \(text)") + } +} +if let first = cellTextByID.keys.min(), let before = cellTextByID[first] { + let after = await scalar("SELECT TZ FROM TP_TZR_T WHERE ID = \(first)") + check(after == before, "after the ALTER, row \(first) still reads \(after ?? "NULL")") +} + +print("\n6. Write-back through the writer's masks") +for (row, text) in cellTextByID.sorted(by: { $0.key < $1.key }) { + let matched = await scalar("SELECT COUNT(*) FROM TP_TZR_T WHERE ID = \(row) AND TZ = \(parsed(text))") + if row == overlapRow { + let resolved = await scalar("SELECT TO_CHAR(\(parsed(text)), 'YYYY-MM-DD HH24:MI:SS TZR TZD') FROM DUAL") + info("the repeated hour: \(text) matches \(matched ?? "no") stored row and resolves to \(resolved ?? "an error")") + continue + } + check(matched == "1", "\(text) parses back to the stored instant") + guard !text.hasSuffix("+00:00") else { continue } + let kept = await scalar("SELECT EXTRACT(TIMEZONE_REGION FROM \(parsed(text))) FROM DUAL") + check(kept == regionByID[row], "\(text) keeps its region: \(kept ?? "NULL")") +} + +let spellings: [(text: String, literal: String)] = [ + ("2024-01-02 03:04:05.123456-05:30", "TIMESTAMP '2024-01-02 03:04:05.123456 -05:30'"), + ("2000-01-01 00:00:00.000000+00:00", "TIMESTAMP '2000-01-01 00:00:00 +00:00'"), + ("2000-01-01 00:00:00", "FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', SESSIONTIMEZONE)"), + ("2024-01-02 03:04:05.500000 Europe/Paris", "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), +] +await run("DROP TABLE TP_TZR_L PURGE") +_ = await result("CREATE TABLE TP_TZR_L (ID NUMBER PRIMARY KEY, L TIMESTAMP(6) WITH LOCAL TIME ZONE)") +for (index, spelling) in spellings.enumerated() { + let compared = await attempt("SELECT CASE WHEN \(parsed(spelling.text)) = \(spelling.literal) THEN 1 ELSE 0 END FROM DUAL") + let equal = compared.result?.rows.first?.first?.stringValue + check(equal == "1", "TIMESTAMP WITH TIME ZONE: '\(spelling.text)' parses to \(spelling.literal)\(compared.failure.map { ": \($0.prefix(100))" } ?? "")") + let inserted = await attempt("INSERT INTO TP_TZR_L VALUES (\(index), \(parsed(spelling.text)))") + var stored: String? + if inserted.failure == nil { + stored = await scalar("SELECT COUNT(*) FROM TP_TZR_L WHERE ID = \(index) AND L = \(spelling.literal)") + } + check(stored == "1", "TIMESTAMP WITH LOCAL TIME ZONE: '\(spelling.text)' writes \(spelling.literal)\(inserted.failure.map { ": \($0.prefix(100))" } ?? "")") +} + +await run("DROP TABLE TP_TZR_T PURGE") +await run("DROP TABLE TP_TZR_L PURGE") +try? await raw.close() +print(disagreements == 0 ? "\nall checks hold" : "\n\(disagreements) disagreement(s)") +exit(disagreements == 0 ? 0 : 1) +EOF + +echo "Building the probe against Packages/TableProOracle (the first build takes a minute)" +(cd "$WORK" && swift build -c debug > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -20 >&2 + exit 3 +} +"$WORK/.build/debug/Probe" "$HOST" "$PORT" "$SERVICE" "$USER_NAME" "$PASSWORD" "$MASK" "$OFFSET_MASK" 2> /dev/null