diff --git a/go.mod b/go.mod index 3600c07..5420330 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/charmbracelet/harmonica v0.2.0 github.com/charmbracelet/ultraviolet v0.0.0-20260525132238-948f4557a654 github.com/charmbracelet/x/ansi v0.11.7 - github.com/extend-hq/extend-go-sdk v0.6.0 + github.com/extend-hq/extend-go-sdk v0.10.0 github.com/itchyny/gojq v0.12.19 github.com/mattn/go-isatty v0.0.21 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index e72756e..adee8ed 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/extend-hq/extend-go-sdk v0.6.0 h1:ZP0TLd77YtFyD8bMxYdpWIsymuJpYdpiFcH+QTtN6vQ= -github.com/extend-hq/extend-go-sdk v0.6.0/go.mod h1:OGwNG071bADLZ/BicWu/TLYn+QGELr/r051eECkfcn0= +github.com/extend-hq/extend-go-sdk v0.10.0 h1:Hv7RlnN0adxWsrncZW9yKrEx2VSxVBH5f+zFQW7uDa4= +github.com/extend-hq/extend-go-sdk v0.10.0/go.mod h1:OGwNG071bADLZ/BicWu/TLYn+QGELr/r051eECkfcn0= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/internal/cli/batch.go b/internal/cli/batch.go index 6ccc6c9..1572942 100644 --- a/internal/cli/batch.go +++ b/internal/cli/batch.go @@ -12,6 +12,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -269,10 +270,13 @@ contained runs with ` + "`extend extract runs list --batch `" + `.`, } extractor.OverrideConfig = &override } - br, err := prep.Client.ExtractRuns.CreateBatch(cmd.Context(), &extend.ExtractRunsCreateBatchRequest{ + req := &extend.ExtractRunsCreateBatchRequest{ Extractor: extractor, Inputs: items, Priority: priorityPtr(f.priority), + } + br, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.BatchRun, error) { + return prep.Client.ExtractRuns.CreateBatch(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("submit batch: %w", err) @@ -347,10 +351,13 @@ contained runs with ` + "`extend classify runs list --batch `" + `.`, } classifier.OverrideConfig = &override } - br, err := prep.Client.ClassifyRuns.CreateBatch(cmd.Context(), &extend.ClassifyRunsCreateBatchRequest{ + req := &extend.ClassifyRunsCreateBatchRequest{ Classifier: classifier, Inputs: items, Priority: priorityPtr(f.priority), + } + br, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.BatchRun, error) { + return prep.Client.ClassifyRuns.CreateBatch(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("submit batch: %w", err) @@ -423,10 +430,13 @@ contained runs with ` + "`extend split runs list --batch `" + `.`, } splitter.OverrideConfig = &override } - br, err := prep.Client.SplitRuns.CreateBatch(cmd.Context(), &extend.SplitRunsCreateBatchRequest{ + req := &extend.SplitRunsCreateBatchRequest{ Splitter: splitter, Inputs: items, Priority: priorityPtr(f.priority), + } + br, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.BatchRun, error) { + return prep.Client.SplitRuns.CreateBatch(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("submit batch: %w", err) @@ -523,10 +533,13 @@ contained runs with ` + "`extend parse runs list --batch `" + `.`, Metadata: metadataPtr(prep.Metadata), } } - br, err := prep.Client.ParseRuns.CreateBatch(cmd.Context(), &extend.ParseRunsCreateBatchRequest{ + req := &extend.ParseRunsCreateBatchRequest{ Inputs: items, Config: cfg, Priority: priorityPtr(priority), + } + br, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.BatchRun, error) { + return prep.Client.ParseRuns.CreateBatch(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("submit batch: %w", err) @@ -536,7 +549,7 @@ contained runs with ` + "`extend parse runs list --batch `" + `.`, Configure: func(cmd *cobra.Command) { cmd.Flags().StringVar(&filesFrom, "files-from", "", "Path to a file with one input per line (- for stdin)") cmd.Flags().StringVar(&target, "target", "markdown", "Parse target: markdown or spatial") - cmd.Flags().StringVar(&engine, "engine", "", "Engine: parse_performance or parse_light (default: server default)") + cmd.Flags().StringVar(&engine, "engine", "", "Engine: parse_auto, parse_performance, or parse_light (default: server default)") cmd.Flags().StringVar(&engineVersion, "engine-version", "", "Engine version (e.g. latest, 1.0.1, 2.0.0-beta)") cmd.Flags().StringVar(&chunkStrategy, "chunk-strategy", "", "Chunking strategy: page|document|section (none omits chunkingStrategy)") cmd.Flags().IntVar(&chunkMinChars, "chunk-min-chars", 0, "Minimum characters per chunk (server default if 0)") @@ -630,12 +643,15 @@ no 'workflows batches' commands. Track progress with: } items[i] = &extend.WorkflowRunsCreateBatchRequestInputsItem{File: file, Secrets: secretsPtr} } - resp, err := prep.Client.WorkflowRuns.CreateBatch(cmd.Context(), &extend.WorkflowRunsCreateBatchRequest{ + req := &extend.WorkflowRunsCreateBatchRequest{ Workflow: &extend.WorkflowReference{ ID: f.using, Version: versionPtr(f.version), }, Inputs: items, + } + resp, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.WorkflowRunsCreateBatchResponse, error) { + return prep.Client.WorkflowRuns.CreateBatch(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("submit batch: %w", err) diff --git a/internal/cli/classify.go b/internal/cli/classify.go index d6c8a94..ac2db3c 100644 --- a/internal/cli/classify.go +++ b/internal/cli/classify.go @@ -11,6 +11,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -208,7 +209,9 @@ func runClassify(ctx context.Context, app *App, p classifyParams) error { req.Priority = &pr } - run, err := cli.ClassifyRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.ClassifyRun, error) { + return cli.ClassifyRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } diff --git a/internal/cli/configdocs.go b/internal/cli/configdocs.go index b67fdfe..1b2b45a 100644 --- a/internal/cli/configdocs.go +++ b/internal/cli/configdocs.go @@ -13,7 +13,9 @@ package cli // // Source of truth is the pinned extend-go-sdk structs: SplitConfig, // ExtractConfigJSON, ClassifyConfig, WorkflowsCreateRequest.Steps, -// ParseConfigBlockOptions, ParseConfigAdvancedOptions, and EditJSON. When +// ParseConfigBlockOptions, ParseConfigAdvancedOptions, +// EditConfigAdvancedOptions, EditSchemaGenerationConfigAdvancedOptions, +// and EditJSON. When // the SDK is regenerated and a field is added, removed, or renamed, update // the matching catalog here in the same change — drift is a defect, not a // follow-up. The drift guards in configdocs_test.go enforce this. @@ -155,7 +157,9 @@ const workflowStepsFields = ` - steps (array) - the workflow's processing graph - CLASSIFY and SPLIT route with classificationId values. These must match classification IDs from the classifier/splitter config, not type labels. - CONDITIONAL routes with conditionId values matching config.conditions[].id. - - RULE_VALIDATION routes with result values: "pass" or "fail". + - RULE_VALIDATION and VALIDATION route with result values: "pass" or "fail". + - ROUTER routes with branchId values matching config.branches[].id, plus the + reserved "otherwise" route taken when no branch matches. - Conditional and formula fields can reference prior processor outputs as "{{stepName.output.*}}", workflow run metadata as "{{metadata.*}}", and external validation responses as @@ -178,6 +182,11 @@ const workflowStepsFields = ` - steps (array) - the workflow's processing graph { "step": "review", "result": "fail" } ] + "next": [ + { "step": "extract_invoice", "branchId": "is_invoice" }, + { "step": "review", "branchId": "otherwise" } + ] + Processor references and versions: - EXTRACT config references { "extractor": { "id": "ex_xxx", "version": "latest" } }. - CLASSIFY config references { "classifier": { "id": "cl_xxx", "version": "0.1" } }. @@ -220,6 +229,17 @@ const workflowStepsFields = ` - steps (array) - the workflow's processing graph the formula must evaluate to a boolean and may reference any earlier step output, including data fetched with EXTERNAL_DATA_VALIDATION. A common topology routes pass to WEBHOOK_RESPONSE and fail to HUMAN_REVIEW. + - VALIDATION - checks workflow data with config.blocks, all of which must pass; + route on result "pass" or "fail". A block is code or natural language: + { "name": "totals_match", "blockType": "code", + "code": { "language": "javascript", "code": "..." } } + { "name": "looks_legit", "blockType": "semantic", + "semantic": { "promptText": "..." } } + Only JavaScript code runs today; python blocks fail when evaluated. + - ROUTER - evaluates config.branches in order and takes the first branch whose + block returns true, or the "otherwise" route when none does. Each branch is + { "id", "name", "block" }, with a code or semantic block as in VALIDATION; + code blocks must return a boolean. - EXTERNAL_DATA_VALIDATION - posts extraction data to an external HTTP endpoint; config.requestOptions sets url, method, headers, and contentType, and failureBehavior controls failures. Extend also signs requests with the @@ -305,7 +325,10 @@ const parseBlockOptionsFields = ` - figures (object) — enabled; figureImageCl (carry headers across page breaks); cellBlocksEnabled (emit a block per cell); agentic (object: enabled, customInstructions). - text (object) — signatureDetectionEnabled; agentic (object: enabled, customInstructions). - - keyValue (object) — blankFieldFormattingEnabled (emit detected blank fields). + - keyValue (object) — blankFieldFormattingEnabled (emit detected blank fields); + structuredOutputEnabled (return each form region as a typed tree of keys, values, + checkboxes, signatures, and tables under details.form; needs parse_performance + >= 2.0.0, parse_light >= 1.0.0, or parse_auto). - barcodes (object) — readingEnabled (decode barcode values); imageClippingEnabled. - formulas (object) — enabled (detect formulas and include their LaTeX representation).` @@ -320,8 +343,20 @@ const parseAdvancedOptionsFields = ` - pageRotationEnabled (bool) — auto-dete - excelIncludeCellMetadata (bool) — include cell provenance (source refs/formulas, data-cell/data-formula in HTML); advanced mode only. - excelIncludeCellFormatting (bool) — include cell formatting (bold, italic, font/background color, inline HTML styles); advanced mode only. - verticalGroupingThreshold (number 0.1–5.0, default 1.0) — line-grouping sensitivity; --target spatial only. - - returnOcr (object) — words (bool): include word-level raw OCR data in the response. + - returnOcr (object) — words (bool): include word-level raw OCR data in the response; + lines (bool): include line-level OCR data at output.ocr.lines. Independent opt-ins. + - textExtractionMethod ("auto"|"ocr_only") — how PDF text is read: "auto" uses the PDF's + embedded text where it is reliable and OCRs the rest; "ocr_only" OCRs every page. + parse_auto and parse_light >= 1.0.0 only (default "auto"); other engines always OCR. + - embedOcrInPdf (bool) — add the recognized words to the processed PDF as an invisible + text layer, so it can be searched and copied; pages that already have text are + skipped. Download the PDF afterwards with ` + "`extend files download `" + `. - alwaysConvertToPdf (bool) — convert images/Office/HTML to PDF first (enables spatial bboxes). + - htmlSourceAnchors (bool) — HTML input only: give each block a source with the byte range it + came from in the file, instead of polygon/boundingBox. The file must be UTF-8 or + Windows-1252; can't be combined with alwaysConvertToPdf. + - returnSourceNodes (bool) — HTML input only: also return word-level source anchors at + output.source.nodes, like returnOcr.words for PDFs. Requires htmlSourceAnchors. - enrichmentFormat ("xml"|"bracket") — annotation style, e.g. 1 vs [page_number: 1]. - imageConversionQuality ("high"|"medium"|"low") — quality when converting to PDF. - formattingDetection (array) — change-tracking detection; each entry { "type": "change_tracking" }. @@ -366,6 +401,12 @@ each field. Populate the value keys; leave detected structural keys as generated fontSize, fontColor (RGB 0-255), font. - extend_edit:column_width (number) — column width percentage for table fields. - extend_edit:row_heights (array) — row height percentages for array/table fields. + - extend_edit:source_acroform (object) — the PDF's own form field behind this property: + fieldName, fieldType, and optionMap/optionValue for choice fields. Generated for + native fields; keep it unchanged. With advancedOptions.preserveSignatureValidity it + names the existing field to fill. + - extend:descriptions (array) — human-readable labels for the enum values, in the same + order. Generated for choice fields; keep it unchanged. Standard JSON Schema keys carry through: description, properties, items, enum, required, additionalProperties, and maxItems (row cap for array/table fields). @@ -373,6 +414,32 @@ Root-level conditional form logic is supported with dependentRequired, if/then/else, allOf, oneOf, anyOf, and not. Conditional clauses do not accept extend_edit:* keys.` +// editAdvancedOptionsFields documents the EditConfigAdvancedOptions object +// (edit --advanced-options). +const editAdvancedOptionsFields = ` flattenPdf bool Make the filled form non-editable (server default: true). + nativeFieldsOnly bool Only use embedded AcroForm fields; set false to also detect fields via vision. + tableParsingEnabled bool Parse table regions as arrays of objects so their cells can be filled. + radioEnumsEnabled bool Model a radio-button group as a single-choice enum so only one option fills. + conditionalGenerationEnabled bool When the server generates the schema, add the form's stated requirements + as conditional rules; values that break them fail the run. + advancedSchemaRefinementEnabled bool When the server generates the schema, organize field names and descriptions; + instructions can nest fields or merge options into one choice (adds latency). + Edit 1.0.0 or later; omitted uses the organization's default (off). + preserveSignatureValidity bool Try to keep existing digital signatures valid: fill only existing fields, matched + by extend_edit:source_acroform.fieldName; flattenPdf then defaults to false, and + flattenPdf true fails on signed PDFs. Edit 1.0.0-beta or later.` + +// formDetectionAdvancedOptionsFields documents the +// EditSchemaGenerationConfigAdvancedOptions object (detect-form +// --advanced-options). +const formDetectionAdvancedOptionsFields = ` nativeFieldsOnly bool Only use embedded AcroForm fields; set false to also detect fields via vision. + tableParsingEnabled bool Parse table regions as arrays of objects. + radioEnumsEnabled bool Model a radio-button group as a single-choice enum. + conditionalGenerationEnabled bool Add the form's stated requirements as root-level conditional rules. + advancedSchemaRefinementEnabled bool Organize field names and descriptions; instructions can nest fields or + merge checkbox/radio options into one choice (adds latency). Edit 1.0.0 or + later; omitted uses the organization's default (off).` + const editOutputDoc = `Edit output: - output.editedFile.id — Extend file ID for the completed PDF; reusable as input diff --git a/internal/cli/configdocs_test.go b/internal/cli/configdocs_test.go index dc7d896..fbbdefc 100644 --- a/internal/cli/configdocs_test.go +++ b/internal/cli/configdocs_test.go @@ -74,6 +74,8 @@ func TestConfigCatalogsCoverSDKFields(t *testing.T) { // Parse --advanced-options: top-level plus the returnOcr leaf. {"ParseConfigAdvancedOptions", reflect.TypeOf(extend.ParseConfigAdvancedOptions{}), parseAdvancedOptionsFields}, {"ParseConfigAdvancedOptionsReturnOcr", reflect.TypeOf(extend.ParseConfigAdvancedOptionsReturnOcr{}), parseAdvancedOptionsFields}, + {"EditConfigAdvancedOptions", reflect.TypeOf(extend.EditConfigAdvancedOptions{}), editAdvancedOptionsFields}, + {"EditSchemaGenerationConfigAdvancedOptions", reflect.TypeOf(extend.EditSchemaGenerationConfigAdvancedOptions{}), formDetectionAdvancedOptionsFields}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/cli/detectform.go b/internal/cli/detectform.go index ff5f455..56cc104 100644 --- a/internal/cli/detectform.go +++ b/internal/cli/detectform.go @@ -10,6 +10,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -63,9 +64,7 @@ run object reports the resolved exact version under config. Detection options ride in --advanced-options as a JSON object (omitted fields use the server default): - nativeFieldsOnly bool Only use embedded AcroForm fields; set false to also detect fields via vision. - tableParsingEnabled bool Parse table regions as arrays of objects. - radioEnumsEnabled bool Model a radio-button group as a single-choice enum. +` + formDetectionAdvancedOptionsFields + ` ` + formDetectionOutputDoc + ` @@ -129,9 +128,12 @@ fields use the server default): } cfg.InputSchema = &schema } - run, err := cli.FormDetectionRuns.Create(cmd.Context(), &extend.FormDetectionRunsCreateRequest{ + req := &extend.FormDetectionRunsCreateRequest{ File: file, Config: cfg, + } + run, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.FormDetectionRun, error) { + return cli.FormDetectionRuns.Create(cmd.Context(), req, opts...) }) if err != nil { return fmt.Errorf("create run: %w", err) @@ -156,7 +158,7 @@ fields use the server default): return renderWithDefault(app, final, output.FormatJSON) }, Configure: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&advancedOptionsPath, "advanced-options", "", "Detection options as a JSON object: nativeFieldsOnly, tableParsingEnabled, radioEnumsEnabled. Source: inline JSON, path, file:// URI, or '-' for stdin. Omitted fields use the server default.") + cmd.Flags().StringVar(&advancedOptionsPath, "advanced-options", "", "Detection options as a JSON object: nativeFieldsOnly, tableParsingEnabled, radioEnumsEnabled, conditionalGenerationEnabled, advancedSchemaRefinementEnabled. Source: inline JSON, path, file:// URI, or '-' for stdin. Omitted fields use the server default.") cmd.Flags().StringVar(&engineVersion, "engine-version", "", "Edit engine version (e.g. latest, 0.0.1, 1.0.0-beta). Omit for the server default; the run reports the resolved exact version.") cmd.Flags().StringVar(&instructions, "instructions", "", "Free-form instructions to guide schema generation") cmd.Flags().StringVar(&inputSchemaPath, "input-schema", "", "Starting-point JSON Schema (overlaid by detection). Source: inline JSON, path, file:// URI, or '-' for stdin.") diff --git a/internal/cli/edit.go b/internal/cli/edit.go index db034e9..1428599 100644 --- a/internal/cli/edit.go +++ b/internal/cli/edit.go @@ -12,6 +12,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -96,10 +97,7 @@ Config fields: version). Pass through --engine-version; omitted runs use the server default. The run object reports the resolved exact version under config. - flattenPdf bool Make the filled form non-editable (server default: true). - nativeFieldsOnly bool Only use embedded AcroForm fields; set false to also detect fields via vision. - tableParsingEnabled bool Parse table regions as arrays of objects so their cells can be filled. - radioEnumsEnabled bool Model a radio-button group as a single-choice enum so only one option fills. +` + editAdvancedOptionsFields + ` Prefer schema-driven fills for production/repeated forms. Natural-language instructions are fastest for prototypes; schemas with extend_edit:value are @@ -122,7 +120,7 @@ the output must remain editable. "--schema and --instructions can be combined; for simple fills, --instructions alone is enough.", "Populate values per the shape emitted by 'extend detect-form' — do not invent field names; inspect the generated schema first.", "--output-file '-' streams the filled PDF to stdout; combine with redirection.", - "Detection toggles (flattenPdf/nativeFieldsOnly/tableParsingEnabled/radioEnumsEnabled) go in --advanced-options JSON; omitted fields use the server default.", + "Options such as flattenPdf, nativeFieldsOnly and preserveSignatureValidity go in --advanced-options JSON; omitted fields use the server default.", "Edit runs cannot have a CANCELLED status; only FAILED or PROCESSED.", }, SeeAlso: []string{"detect-form", "edit runs watch", "edit runs get", "files download"}, @@ -156,7 +154,7 @@ the output must remain editable. cmd.Flags().StringVar(&schemaPath, "schema", "", "Inline JSON, path, file:// URI, or '-' for a schema with values populated per the shape emitted by 'extend detect-form'. Omit to let the server auto-detect form fields.") cmd.Flags().StringVar(&instructions, "instructions", "", "Free-form prose values and rules (e.g. \"name is Acme Corp; format dates as MM/DD/YYYY\"). Use alone for simple fills, or alongside --schema for fills that need conditional or formatting guidance the schema cannot express.") cmd.Flags().StringVar(&schemaGenInstructions, "schema-instructions", "", "Free-form prose applied only to the schema-generation step when --schema is omitted (which fields to include, how to interpret ambiguous layouts).") - cmd.Flags().StringVar(&advancedOptionsPath, "advanced-options", "", "Detection options as a JSON object: flattenPdf, nativeFieldsOnly, tableParsingEnabled, radioEnumsEnabled. Source: inline JSON, path, file:// URI, or '-' for stdin. Omitted fields use the server default.") + cmd.Flags().StringVar(&advancedOptionsPath, "advanced-options", "", "Edit options as a JSON object: flattenPdf, nativeFieldsOnly, tableParsingEnabled, radioEnumsEnabled, conditionalGenerationEnabled, advancedSchemaRefinementEnabled, preserveSignatureValidity. Source: inline JSON, path, file:// URI, or '-' for stdin. Omitted fields use the server default.") cmd.Flags().StringVar(&engineVersion, "engine-version", "", "Edit engine version (e.g. latest, 0.0.1, 1.0.0-beta). Omit for the server default; the run reports the resolved exact version.") cmd.Flags().StringVarP(&outputFile, "output-file", "O", "", "Path to write the filled PDF to (auto-downloads); '-' for stdout. Default: leave the PDF on the server; fetch later with 'extend files download '.") cmd.Flags().StringVar(&password, "password", "", "Password for a password-protected PDF (URL inputs only)") @@ -234,7 +232,9 @@ func runEdit(ctx context.Context, app *App, p editParams) error { Config: cfg, } - run, err := cli.EditRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.EditRun, error) { + return cli.EditRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } diff --git a/internal/cli/evaluations.go b/internal/cli/evaluations.go index 502384d..02aae13 100644 --- a/internal/cli/evaluations.go +++ b/internal/cli/evaluations.go @@ -8,7 +8,9 @@ import ( "github.com/spf13/cobra" extend "github.com/extend-hq/extend-go-sdk" + "github.com/extend-hq/extend-go-sdk/option" + "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" ) @@ -607,7 +609,9 @@ with 'extend evaluations runs get'.`, if err != nil { return err } - run, err := cli.EvaluationSetRuns.Create(cmd.Context(), &req) + run, err := extendx.StartOnce(cmd.Context(), func(opts ...option.RequestOption) (*extend.EvaluationSetRun, error) { + return cli.EvaluationSetRuns.Create(cmd.Context(), &req, opts...) + }) if err != nil { return err } diff --git a/internal/cli/extract.go b/internal/cli/extract.go index ac41843..98fa820 100644 --- a/internal/cli/extract.go +++ b/internal/cli/extract.go @@ -11,6 +11,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -216,7 +217,9 @@ func runExtract(ctx context.Context, app *App, p extractParams) error { req.Priority = &pr } - run, err := cli.ExtractRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.ExtractRun, error) { + return cli.ExtractRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } diff --git a/internal/cli/parse.go b/internal/cli/parse.go index 053cafa..05c1c45 100644 --- a/internal/cli/parse.go +++ b/internal/cli/parse.go @@ -12,6 +12,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -110,7 +111,7 @@ knobs in the same forms. }, Configure: func(cmd *cobra.Command) { cmd.Flags().StringVar(&target, "target", "markdown", "Parse target: markdown or spatial") - cmd.Flags().StringVar(&engine, "engine", "", "Engine: parse_performance or parse_light (default: server default)") + cmd.Flags().StringVar(&engine, "engine", "", "Engine: parse_auto, parse_performance, or parse_light (default: server default)") cmd.Flags().StringVar(&engineVersion, "engine-version", "", "Engine version (e.g. latest, 1.0.1, 2.0.0-beta)") cmd.Flags().StringVar(&chunkStrategy, "chunk-strategy", "", "Chunking strategy: page|document|section (none omits chunkingStrategy)") cmd.Flags().IntVar(&chunkMinChars, "chunk-min-chars", 0, "Minimum characters per chunk (server default if 0)") @@ -172,7 +173,9 @@ func runParse(ctx context.Context, app *App, p parseParams) error { req.Metadata = &md } - run, err := cli.ParseRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.ParseRun, error) { + return cli.ParseRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } @@ -222,10 +225,9 @@ func buildParseConfig(p parseParams) (*extend.ParseConfig, error) { cfg.Target = &t } if p.engine != "" { - e, err := extend.NewParseConfigEngineFromString(p.engine) - if err != nil { - return nil, fmt.Errorf("--engine: %w", err) - } + // Passed through unchecked so an engine the API adds works before + // the SDK lists it; the API rejects unknown names itself. + e := extend.ParseConfigEngine(p.engine) cfg.Engine = &e } if p.engineVersion != "" { diff --git a/internal/cli/render.go b/internal/cli/render.go index 045f268..890cd84 100644 --- a/internal/cli/render.go +++ b/internal/cli/render.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/json" "fmt" "reflect" "strings" @@ -20,12 +21,39 @@ func renderWithDefault(app *App, payload any, defaultFormat output.Format) error } f = parsed } - return output.Render(app.IO.Out, f, payload, + return output.Render(app.IO.Out, f, apiJSON(payload), output.WithJQ(app.JQ), output.WithPretty(app.IO.IsStdoutTTY()), ) } +// sdkObject matches the types the SDK decodes API responses into. Their +// String method returns the JSON the API sent. +type sdkObject interface { + GetExtraProperties() map[string]interface{} + String() string +} + +// apiJSON swaps an SDK response object for the JSON the API sent, so +// fields the SDK doesn't model still reach -o json, -o yaml and --jq. +// Other values, and objects the CLI built itself, come back unchanged. +func apiJSON(payload any) any { + switch v := payload.(type) { + case []any: + out := make([]any, len(v)) + for i, item := range v { + out[i] = apiJSON(item) + } + return out + case sdkObject: + var decoded any + if err := json.Unmarshal([]byte(v.String()), &decoded); err == nil { + return decoded + } + } + return payload +} + // renderList picks the right renderer for a paginated list result based on // --output: // diff --git a/internal/cli/request_body_test.go b/internal/cli/request_body_test.go new file mode 100644 index 0000000..81595c9 --- /dev/null +++ b/internal/cli/request_body_test.go @@ -0,0 +1,140 @@ +package cli + +import ( + "encoding/json" + "net/http" + "reflect" + "testing" +) + +// The CLI decodes JSON flags into SDK request types and encodes them again. +// These cases pin JSON the SDK used to drop or rewrite on that round trip: +// each one must reach the server with the given value at the given path. +func TestJSONFlagsReachTheServerIntact(t *testing.T) { + cases := []struct { + name string + args []string + path []any + want any + }{ + { + name: "classifier evaluation item", + args: []string{"evaluations", "items", "create", "ev_1", "--from-file", + `{"items":[{"fileId":"file_1","expectedOutput":{"id":"cls_a","type":"invoice"}}]}`}, + path: []any{"items", 0, "expectedOutput"}, + want: map[string]any{"id": "cls_a", "type": "invoice"}, + }, + { + name: "splitter evaluation item", + args: []string{"evaluations", "items", "create", "ev_1", "--from-file", + `{"items":[{"fileId":"file_1","expectedOutput":{"splits":[{"classificationId":"c1","type":"invoice","startPage":1,"endPage":2}]}}]}`}, + path: []any{"items", 0, "expectedOutput", "splits", 0, "classificationId"}, + want: "c1", + }, + { + name: "multi-file evaluation item", + args: []string{"evaluations", "items", "create", "ev_1", "--from-file", + `{"items":[{"fileIds":["file_1","file_2"],"expectedOutput":{"value":{"total":1}}}]}`}, + path: []any{"items", 0, "fileIds"}, + want: []any{"file_1", "file_2"}, + }, + { + name: "classifier output seeded into a workflow run", + args: []string{"workflows", "run", "file_1", "--using", "workflow_1", "--outputs", + `[{"processorId":"cl_1","output":{"id":"cls_a","type":"invoice"}}]`}, + path: []any{"outputs", 0, "output"}, + want: map[string]any{"id": "cls_a", "type": "invoice"}, + }, + { + name: "extractor generated from a file ID", + args: []string{"extractors", "create", "--from-file", + `{"name":"Invoices","generate":{"files":[{"id":"file_1"}]}}`}, + path: []any{"generate", "files", 0}, + want: map[string]any{"id": "file_1"}, + }, + { + name: "structured key-value output", + args: []string{"parse", "file_1", "--wait=false", "--block-options", + `{"keyValue":{"structuredOutputEnabled":true}}`}, + path: []any{"config", "blockOptions", "keyValue", "structuredOutputEnabled"}, + want: true, + }, + { + name: "line-level OCR", + args: []string{"parse", "file_1", "--wait=false", "--advanced-options", + `{"returnOcr":{"lines":true}}`}, + path: []any{"config", "advancedOptions", "returnOcr", "lines"}, + want: true, + }, + { + name: "auto parse engine", + args: []string{"parse", "file_1", "--wait=false", "--engine", "parse_auto"}, + path: []any{"config", "engine"}, + want: "parse_auto", + }, + { + name: "split strict identifier mode", + args: []string{"split", "file_1", "--wait=false", "--config", + `{"splitClassifications":[{"id":"o","type":"other","description":"d"}],"advancedOptions":{"strictIdentifierModeEnabled":true}}`}, + path: []any{"config", "advancedOptions", "strictIdentifierModeEnabled"}, + want: true, + }, + { + name: "edit signature and schema refinement options", + args: []string{"edit", "file_1", "--wait=false", "--advanced-options", + `{"preserveSignatureValidity":true,"advancedSchemaRefinementEnabled":true}`}, + path: []any{"config", "advancedOptions"}, + want: map[string]any{"preserveSignatureValidity": true, "advancedSchemaRefinementEnabled": true}, + }, + { + name: "edit schema with native form field details", + args: []string{"edit", "file_1", "--wait=false", "--schema", + `{"type":"object","properties":{"marital":{"type":["string","null"],"enum":["s","m"],"extend:descriptions":["Single","Married"],"extend_edit:source_acroform":{"fieldName":"Marital1","fieldType":"radio"}}}}`}, + path: []any{"config", "schema", "properties", "marital"}, + want: map[string]any{ + "type": []any{"string", "null"}, + "enum": []any{"s", "m"}, + "extend:descriptions": []any{"Single", "Married"}, + "extend_edit:source_acroform": map[string]any{"fieldName": "Marital1", "fieldType": "radio"}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := newFakeServer(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusBadRequest, map[string]any{"code": "INVALID_REQUEST", "message": "stop here"}) + }) + ta := newTestApp(t, srv) + cmd := findCmd(t, ta.app, tc.args[0]) + cmd.SetArgs(tc.args[1:]) + cmd.SetOut(ta.out) + cmd.SetErr(ta.errOut) + _ = cmd.Execute() + + var body any + if err := json.Unmarshal(srv.lastRequest().Body, &body); err != nil { + t.Fatalf("request body is not JSON: %v\n%s", err, srv.lastRequest().Body) + } + got := body + for _, step := range tc.path { + switch key := step.(type) { + case string: + m, ok := got.(map[string]any) + if !ok { + t.Fatalf("no %q in request body:\n%s", key, srv.lastRequest().Body) + } + got = m[key] + case int: + a, ok := got.([]any) + if !ok || key >= len(a) { + t.Fatalf("no [%d] in request body:\n%s", key, srv.lastRequest().Body) + } + got = a[key] + } + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("sent %v at %v, want %v\nfull body: %s", got, tc.path, tc.want, srv.lastRequest().Body) + } + }) + } +} diff --git a/internal/cli/response_fields_test.go b/internal/cli/response_fields_test.go new file mode 100644 index 0000000..79ea2dd --- /dev/null +++ b/internal/cli/response_fields_test.go @@ -0,0 +1,93 @@ +package cli + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/extend-hq/extend-cli/internal/extendx" +) + +// Fields the SDK doesn't model must still reach -o json and --jq, since +// the API adds response fields between SDK releases. +func TestJSONOutputKeepsFieldsTheSDKDoesNotModel(t *testing.T) { + run := map[string]any{ + "object": "extract_run", + "id": "exr_1", + "status": "PROCESSED", + "failureReason": nil, + "files": []any{ + map[string]any{"object": "file", "id": "file_1", "name": "a.pdf", "parseRunId": "pr_1"}, + }, + "notYetInTheSDK": map[string]any{"mode": "zero"}, + } + + t.Run("get", func(t *testing.T) { + srv := newFakeServer(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, run) + }) + ta := newTestApp(t, srv) + ta.app.Format = "json" + if err := runTypedRunsGet(context.Background(), ta.app, extendx.KindExtract, "exr_1", ""); err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(ta.out.Bytes(), &got); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, ta.out.String()) + } + if _, ok := got["notYetInTheSDK"]; !ok { + t.Errorf("unknown top-level field dropped:\n%s", ta.out.String()) + } + if v, ok := got["failureReason"]; !ok || v != nil { + t.Errorf("explicit null dropped:\n%s", ta.out.String()) + } + file := got["files"].([]any)[0].(map[string]any) + if file["parseRunId"] != "pr_1" { + t.Errorf("unknown nested field dropped:\n%s", ta.out.String()) + } + }) + + t.Run("jq", func(t *testing.T) { + srv := newFakeServer(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, run) + }) + ta := newTestApp(t, srv) + ta.app.Format = "raw" + ta.app.JQ = ".notYetInTheSDK.mode" + if err := runTypedRunsGet(context.Background(), ta.app, extendx.KindExtract, "exr_1", ""); err != nil { + t.Fatal(err) + } + if got := ta.out.String(); got != "zero\n" { + t.Errorf("--jq output = %q, want %q", got, "zero\n") + } + }) + + t.Run("list", func(t *testing.T) { + srv := newFakeServer(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "object": "list", + "data": []any{run}, + "notYetInTheSDK": true, + }) + }) + ta := newTestApp(t, srv) + ta.app.Format = "json" + cmd := findCmd(t, ta.app, "extract", "runs", "list") + cmd.SetArgs(nil) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(ta.out.Bytes(), &got); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, ta.out.String()) + } + if got["notYetInTheSDK"] != true { + t.Errorf("unknown field on the list dropped:\n%s", ta.out.String()) + } + item := got["data"].([]any)[0].(map[string]any) + if _, ok := item["notYetInTheSDK"]; !ok { + t.Errorf("unknown field on a list item dropped:\n%s", ta.out.String()) + } + }) +} diff --git a/internal/cli/run.go b/internal/cli/run.go index 118bc60..a23d32c 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -10,6 +10,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -203,7 +204,9 @@ func runWorkflow(ctx context.Context, app *App, p workflowParams) error { req.Secrets = &secrets } - run, err := cli.WorkflowRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.WorkflowRun, error) { + return cli.WorkflowRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } diff --git a/internal/cli/split.go b/internal/cli/split.go index f4474c1..f55610f 100644 --- a/internal/cli/split.go +++ b/internal/cli/split.go @@ -11,6 +11,7 @@ import ( extend "github.com/extend-hq/extend-go-sdk" sdkclient "github.com/extend-hq/extend-go-sdk/client" + "github.com/extend-hq/extend-go-sdk/option" "github.com/extend-hq/extend-cli/internal/extendx" "github.com/extend-hq/extend-cli/internal/output" @@ -195,7 +196,9 @@ func runSplit(ctx context.Context, app *App, p splitParams) error { req.Priority = &pr } - run, err := cli.SplitRuns.Create(ctx, req) + run, err := extendx.StartOnce(ctx, func(opts ...option.RequestOption) (*extend.SplitRun, error) { + return cli.SplitRuns.Create(ctx, req, opts...) + }) if err != nil { return fmt.Errorf("create run: %w", err) } diff --git a/internal/extendx/start.go b/internal/extendx/start.go new file mode 100644 index 0000000..203ec86 --- /dev/null +++ b/internal/extendx/start.go @@ -0,0 +1,63 @@ +package extendx + +import ( + "context" + "errors" + "net/http" + "strconv" + "time" + + sdkcore "github.com/extend-hq/extend-go-sdk/core" + "github.com/extend-hq/extend-go-sdk/option" +) + +const ( + maxRateLimitRetries = 2 + maxRateLimitDelay = time.Minute +) + +// StartOnce calls an SDK method that starts work on the server, such as +// creating a run or a batch. The SDK's own retries are off for it: after a +// 408 or 5xx the server may already have started the work, and a retry +// would start it again. A 429 is retried after the wait the server asks +// for, since the API turns rate-limited requests away before doing +// anything. +func StartOnce[T any](ctx context.Context, call func(...option.RequestOption) (T, error)) (T, error) { + for attempt := 0; ; attempt++ { + result, err := call(option.WithoutRetries()) + if err == nil || attempt == maxRateLimitRetries { + return result, err + } + delay, ok := rateLimitDelay(err, attempt) + if !ok { + return result, err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + var zero T + return zero, ctx.Err() + case <-timer.C: + } + } +} + +// rateLimitDelay reports whether err is a 429 and how long to wait before +// the next attempt: the server's Retry-After when it sends one, otherwise +// an exponential backoff starting at one second. +func rateLimitDelay(err error, attempt int) (time.Duration, bool) { + var apiErr *sdkcore.APIError + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusTooManyRequests { + return 0, false + } + delay := time.Second << attempt + if v := apiErr.Header.Get("Retry-After"); v != "" { + if secs, perr := strconv.Atoi(v); perr == nil && secs >= 0 { + delay = time.Duration(secs) * time.Second + } else if at, perr := http.ParseTime(v); perr == nil { + delay = max(time.Until(at), 0) + } + } + return min(delay, maxRateLimitDelay), true +} diff --git a/internal/extendx/start_test.go b/internal/extendx/start_test.go new file mode 100644 index 0000000..3e06446 --- /dev/null +++ b/internal/extendx/start_test.go @@ -0,0 +1,79 @@ +package extendx + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + extend "github.com/extend-hq/extend-go-sdk" + "github.com/extend-hq/extend-go-sdk/option" +) + +// startParseRun creates a parse run through StartOnce against a server +// that answers with the given statuses in order, and reports how many +// requests reached it. +func startParseRun(t *testing.T, statuses ...int) (int32, error) { + t.Helper() + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := int(calls.Add(1)) + status := statuses[min(n, len(statuses))-1] + w.Header().Set("Content-Type", "application/json") + if status == http.StatusTooManyRequests { + w.Header().Set("Retry-After", "0") + } + w.WriteHeader(status) + if status == http.StatusOK { + _, _ = w.Write([]byte(`{"object":"parse_run","id":"pr_1","status":"PROCESSING"}`)) + return + } + _, _ = w.Write([]byte(`{"code":"INTERNAL_ERROR","message":"boom","retryable":true}`)) + })) + t.Cleanup(srv.Close) + + c, err := NewClient(Config{APIKey: "k", BaseURL: srv.URL}) + if err != nil { + t.Fatal(err) + } + file, err := BuildParseFile(FileRef{ID: "file_1"}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + _, err = StartOnce(ctx, func(opts ...option.RequestOption) (*extend.ParseRun, error) { + return c.ParseRuns.Create(ctx, &extend.ParseRunsCreateRequest{File: file}, opts...) + }) + return calls.Load(), err +} + +func TestStartOnce_DoesNotRetryServerErrors(t *testing.T) { + calls, err := startParseRun(t, http.StatusBadGateway, http.StatusOK) + if err == nil { + t.Fatal("expected the 502 to be returned") + } + if calls != 1 { + t.Errorf("server saw %d requests, want 1: a retried 5xx can start the run twice", calls) + } +} + +func TestStartOnce_RetriesRateLimits(t *testing.T) { + calls, err := startParseRun(t, http.StatusTooManyRequests, http.StatusOK) + if err != nil { + t.Fatalf("expected success after the 429, got %v", err) + } + if calls != 2 { + t.Errorf("server saw %d requests, want 2", calls) + } +} + +func TestStartOnce_GivesUpAfterTwoRateLimitRetries(t *testing.T) { + calls, err := startParseRun(t, http.StatusTooManyRequests) + if err == nil { + t.Fatal("expected the last 429 to be returned") + } + if calls != 3 { + t.Errorf("server saw %d requests, want 3", calls) + } +}