diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 652f257..335fc56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: rust: [stable, beta] include: - os: ubuntu-latest - rust: nightly-2026-04-16 + rust: nightly steps: - name: Checkout sources uses: actions/checkout@v6 diff --git a/.github/workflows/update-changelog.yml b/.github/workflows/update-changelog.yml new file mode 100644 index 0000000..1d9fb59 --- /dev/null +++ b/.github/workflows/update-changelog.yml @@ -0,0 +1,129 @@ +name: Update Changelog + +on: + pull_request: + types: [opened, edited, synchronize] + +permissions: + contents: write + pull-requests: write + +jobs: + update-changelog: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Check if CHANGELOG already modified in this PR + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + FILES=$(gh pr diff ${{ github.event.pull_request.number }} --name-only) + if echo "$FILES" | grep -qx "CHANGELOG.md"; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Update CHANGELOG + if: steps.check.outputs.changed == 'false' + run: | + PR_TITLE="${{ github.event.pull_request.title }}" + PR_NUMBER="${{ github.event.pull_request.number }}" + REPO="${{ github.repository }}" + EVENT="${{ github.event.action }}" + + FORMATTED_TITLE="$(echo "$PR_TITLE" | sed -E 's/^([a-zA-Z0-9_-]+):/*\1*:/')" + CHANGELOG_ENTRY="- ${FORMATTED_TITLE} ([#${PR_NUMBER}](https://github.com/${REPO}/pull/${PR_NUMBER}))" + + # Pattern to match an existing entry for this PR number (anywhere in the file) + PR_PATTERN="(#${PR_NUMBER})" + + if [ "$EVENT" = "opened" ]; then + # On open: only insert if no entry for this PR number already exists + if grep -qF "$PR_PATTERN" CHANGELOG.md; then + echo "Changelog entry for #${PR_NUMBER} already exists, skipping." + else + awk -v entry="$CHANGELOG_ENTRY" ' + /^## \[Unreleased\]$/ { + print + print "" + print entry + skip_blank=1 + next + } + skip_blank && /^$/ { skip_blank=0; next } + { skip_blank=0; print } + ' CHANGELOG.md > CHANGELOG.md.tmp + mv CHANGELOG.md.tmp CHANGELOG.md + fi + + elif [ "$EVENT" = "edited" ]; then + # On edit: replace the existing entry, or insert if it doesn't exist yet + if grep -qF "$PR_PATTERN" CHANGELOG.md; then + awk -v entry="$CHANGELOG_ENTRY" -v pattern="$PR_PATTERN" ' + index($0, pattern) { print entry; next } + { print } + ' CHANGELOG.md > CHANGELOG.md.tmp + mv CHANGELOG.md.tmp CHANGELOG.md + else + echo "No existing entry for #${PR_NUMBER}, inserting under [Unreleased]." + awk -v entry="$CHANGELOG_ENTRY" ' + /^## \[Unreleased\]$/ { + print + print "" + print entry + skip_blank=1 + next + } + skip_blank && /^$/ { skip_blank=0; next } + { skip_blank=0; print } + ' CHANGELOG.md > CHANGELOG.md.tmp + mv CHANGELOG.md.tmp CHANGELOG.md + fi + fi + + - name: Commit via GitHub API + if: steps.check.outputs.changed == 'false' + env: + GH_TOKEN: ${{ github.token }} + run: | + BRANCH="${{ github.event.pull_request.head.ref }}" + REPO="${{ github.repository }}" + + if git diff --quiet CHANGELOG.md; then + echo "No changes to commit." + exit 0 + fi + + LATEST_SHA=$(gh api "repos/${REPO}/git/refs/heads/${BRANCH}" --jq .object.sha) + BASE_TREE=$(gh api "repos/${REPO}/git/commits/${LATEST_SHA}" --jq .tree.sha) + + BLOB_SHA=$(gh api "repos/${REPO}/git/blobs" \ + -f content="$(base64 -w0 CHANGELOG.md)" \ + -f encoding="base64" \ + --jq .sha) + + NEW_TREE=$(gh api "repos/${REPO}/git/trees" \ + -f base_tree="$BASE_TREE" \ + -f "tree[][path]=CHANGELOG.md" \ + -f "tree[][mode]=100644" \ + -f "tree[][type]=blob" \ + -f "tree[][sha]=$BLOB_SHA" \ + --jq .sha) + + NEW_COMMIT=$(gh api "repos/${REPO}/git/commits" \ + -f message="docs: update CHANGELOG for PR #${{ github.event.pull_request.number }}" \ + -f tree="$NEW_TREE" \ + -f "parents[]=$LATEST_SHA" \ + --jq .sha) + + gh api "repos/${REPO}/git/refs/heads/${BRANCH}" \ + -X PATCH \ + -f sha="$NEW_COMMIT" \ + -F force=false diff --git a/.sqlx/query-066be74a69b4e3f45a1e7c0cd85896b26ba97059de53ebb5773d6d9d4dfbbec7.json b/.sqlx/query-066be74a69b4e3f45a1e7c0cd85896b26ba97059de53ebb5773d6d9d4dfbbec7.json deleted file mode 100644 index 27de918..0000000 --- a/.sqlx/query-066be74a69b4e3f45a1e7c0cd85896b26ba97059de53ebb5773d6d9d4dfbbec7.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO\n apalis.jobs (\n id,\n job_type,\n job,\n status,\n attempts,\n max_attempts,\n run_at,\n priority,\n metadata,\n idempotency_key\n )\nSELECT\n unnest($1::text[]) as id,\n $2::text as job_type,\n unnest($3::bytea[]) as job,\n 'Pending' as status,\n 0 as attempts,\n unnest($4::integer []) as max_attempts,\n unnest($5::timestamptz []) as run_at,\n unnest($6::integer []) as priority,\n unnest($7::jsonb []) as metadata,\n unnest($8::text []) as idempotency_key\n", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "TextArray", - "Text", - "ByteaArray", - "Int4Array", - "TimestamptzArray", - "Int4Array", - "JsonbArray", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "066be74a69b4e3f45a1e7c0cd85896b26ba97059de53ebb5773d6d9d4dfbbec7" -} diff --git a/.sqlx/query-06e70d80d4e7d8f96590795ed48fa43f0c11121df0630e4ac7c5cb937048648c.json b/.sqlx/query-06e70d80d4e7d8f96590795ed48fa43f0c11121df0630e4ac7c5cb937048648c.json deleted file mode 100644 index d4db347..0000000 --- a/.sqlx/query-06e70d80d4e7d8f96590795ed48fa43f0c11121df0630e4ac7c5cb937048648c.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n apalis.jobs\nSET\n status = $4,\n attempts = $2,\n last_result = $3,\n done_at = NOW()\nWHERE\n id = $1\n AND lock_by = $5\n", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int4", - "Jsonb", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "06e70d80d4e7d8f96590795ed48fa43f0c11121df0630e4ac7c5cb937048648c" -} diff --git a/.sqlx/query-1989cc0be6f389d4211ebd23cfa961105f4df7c023b50b0da3acae8881d4d010.json b/.sqlx/query-1989cc0be6f389d4211ebd23cfa961105f4df7c023b50b0da3acae8881d4d010.json deleted file mode 100644 index 33472b0..0000000 --- a/.sqlx/query-1989cc0be6f389d4211ebd23cfa961105f4df7c023b50b0da3acae8881d4d010.json +++ /dev/null @@ -1,185 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE apalis.jobs\nSET \n status = 'Queued',\n lock_at = now(),\n lock_by = $2\nWHERE \n status = 'Pending'\n AND run_at < now()\n AND id = ANY($1)\nRETURNING *;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job", - "type_info": "Bytea", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "job" - } - } - }, - { - "ordinal": 1, - "name": "id", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "id" - } - } - }, - { - "ordinal": 2, - "name": "job_type", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "job_type" - } - } - }, - { - "ordinal": 3, - "name": "status", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "status" - } - } - }, - { - "ordinal": 4, - "name": "attempts", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "attempts" - } - } - }, - { - "ordinal": 5, - "name": "max_attempts", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "max_attempts" - } - } - }, - { - "ordinal": 6, - "name": "run_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "run_at" - } - } - }, - { - "ordinal": 7, - "name": "last_result", - "type_info": "Jsonb", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "last_result" - } - } - }, - { - "ordinal": 8, - "name": "lock_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "lock_at" - } - } - }, - { - "ordinal": 9, - "name": "lock_by", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "lock_by" - } - } - }, - { - "ordinal": 10, - "name": "done_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "done_at" - } - } - }, - { - "ordinal": 11, - "name": "priority", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "priority" - } - } - }, - { - "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "metadata" - } - } - }, - { - "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "idempotency_key" - } - } - } - ], - "parameters": { - "Left": [ - "TextArray", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "1989cc0be6f389d4211ebd23cfa961105f4df7c023b50b0da3acae8881d4d010" -} diff --git a/.sqlx/query-23e8905ae8a0b00ab7f157dd83dd15952577dd54443b3445275e66d5fa7419f8.json b/.sqlx/query-23e8905ae8a0b00ab7f157dd83dd15952577dd54443b3445275e66d5fa7419f8.json deleted file mode 100644 index a487202..0000000 --- a/.sqlx/query-23e8905ae8a0b00ab7f157dd83dd15952577dd54443b3445275e66d5fa7419f8.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n apalis.workers\nSET\n last_seen = NOW()\nWHERE\n id = $1 AND worker_type = $2;\n", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "23e8905ae8a0b00ab7f157dd83dd15952577dd54443b3445275e66d5fa7419f8" -} diff --git a/.sqlx/query-4381cd6827de12b4c259cda0893d13c07e873a1f3b7f10af34156921eedb2f3c.json b/.sqlx/query-4381cd6827de12b4c259cda0893d13c07e873a1f3b7f10af34156921eedb2f3c.json new file mode 100644 index 0000000..4a9f20d --- /dev/null +++ b/.sqlx/query-4381cd6827de12b4c259cda0893d13c07e873a1f3b7f10af34156921eedb2f3c.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO apalis.jobs (\n id,\n job_type,\n job,\n status,\n attempts,\n max_attempts,\n run_at,\n priority,\n metadata,\n idempotency_key\n)\nSELECT\n unnest($1::text[]) AS id,\n $2::text AS job_type,\n unnest($3::bytea[]) AS job,\n 'Pending' AS status,\n 0 AS attempts,\n unnest($4::integer[]) AS max_attempts,\n to_timestamp(unnest($5::bigint[])) AS run_at,\n unnest($6::integer[]) AS priority,\n unnest($7::hstore[]) AS metadata,\n unnest($8::text[]) AS idempotency_key\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Text", + "ByteaArray", + "Int4Array", + "Int8Array", + "Int4Array", + { + "Custom": { + "name": "hstore[]", + "kind": { + "Array": { + "Custom": { + "name": "hstore", + "kind": "Simple" + } + } + } + } + }, + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "4381cd6827de12b4c259cda0893d13c07e873a1f3b7f10af34156921eedb2f3c" +} diff --git a/.sqlx/query-4aaacfe4160824ce6c68a5e26698812af44e47e9e5032475ca83c9a04f039256.json b/.sqlx/query-4aaacfe4160824ce6c68a5e26698812af44e47e9e5032475ca83c9a04f039256.json new file mode 100644 index 0000000..f0c1bee --- /dev/null +++ b/.sqlx/query-4aaacfe4160824ce6c68a5e26698812af44e47e9e5032475ca83c9a04f039256.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE\n apalis.jobs\nSET\n status = 'Running',\n lock_at = now(),\n lock_by = $2\nWHERE\n (\n status = 'Pending'\n OR status = 'Queued'\n OR (\n status = 'Failed'\n AND attempts < max_attempts\n )\n )\n AND run_at < now()\n AND id = ANY($1);\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4aaacfe4160824ce6c68a5e26698812af44e47e9e5032475ca83c9a04f039256" +} diff --git a/.sqlx/query-5d7ccd8d4267874312eb02ff9b8ff0de07d7deb30bbe797ce1128d2d15e3a35d.json b/.sqlx/query-5d7ccd8d4267874312eb02ff9b8ff0de07d7deb30bbe797ce1128d2d15e3a35d.json deleted file mode 100644 index a50a8b9..0000000 --- a/.sqlx/query-5d7ccd8d4267874312eb02ff9b8ff0de07d7deb30bbe797ce1128d2d15e3a35d.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n apalis.jobs\nSET\n status = 'Pending',\n done_at = NULL,\n lock_by = NULL,\n lock_at = NULL,\n attempts = attempts + 1,\n last_result = '{\"Err\": \"Re-enqueued due to worker heartbeat timeout.\"}'\nWHERE\n id IN (\n SELECT\n jobs.id\n FROM\n apalis.jobs\n INNER JOIN apalis.workers ON lock_by = workers.id\n WHERE\n (\n status = 'Running'\n OR status = 'Queued'\n )\n AND NOW() - apalis.workers.last_seen >= $1\n AND apalis.workers.worker_type = $2\n );\n", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Interval", - "Text" - ] - }, - "nullable": [] - }, - "hash": "5d7ccd8d4267874312eb02ff9b8ff0de07d7deb30bbe797ce1128d2d15e3a35d" -} diff --git a/.sqlx/query-37cf19d29005b40bb20786f9bfc0518adf4d213b30c7f0a0848dc54e9e3f6852.json b/.sqlx/query-657d35e8e30d518b3cf3f95af2f4ee5fb1a20fface444233a18f41c4c13426c0.json similarity index 53% rename from .sqlx/query-37cf19d29005b40bb20786f9bfc0518adf4d213b30c7f0a0848dc54e9e3f6852.json rename to .sqlx/query-657d35e8e30d518b3cf3f95af2f4ee5fb1a20fface444233a18f41c4c13426c0.json index d56571b..30e2dfd 100644 --- a/.sqlx/query-37cf19d29005b40bb20786f9bfc0518adf4d213b30c7f0a0848dc54e9e3f6852.json +++ b/.sqlx/query-657d35e8e30d518b3cf3f95af2f4ee5fb1a20fface444233a18f41c4c13426c0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n status,\n last_result AS result\nFROM\n apalis.jobs\nWHERE\n id IN (\n SELECT\n value::text\n FROM\n jsonb_array_elements_text($1) AS value\n )\n AND (\n status = 'Done'\n OR (\n status = 'Failed'\n AND attempts >= max_attempts\n )\n OR status = 'Killed'\n );\n", + "query": "SELECT\n id,\n status,\n attempts as attempt, \n last_result AS result\nFROM\n apalis.jobs\nWHERE\n id IN (\n SELECT\n value::text\n FROM\n jsonb_array_elements_text($1) AS value\n )\n AND (\n status = 'Done'\n OR (\n status = 'Failed'\n AND attempts >= max_attempts\n )\n OR status = 'Killed'\n );\n", "describe": { "columns": [ { @@ -27,6 +27,17 @@ }, { "ordinal": 2, + "name": "attempt", + "type_info": "Int4", + "origin": { + "Table": { + "table": "apalis.jobs", + "name": "attempts" + } + } + }, + { + "ordinal": 3, "name": "result", "type_info": "Jsonb", "origin": { @@ -43,10 +54,11 @@ ] }, "nullable": [ + false, false, false, true ] }, - "hash": "37cf19d29005b40bb20786f9bfc0518adf4d213b30c7f0a0848dc54e9e3f6852" + "hash": "657d35e8e30d518b3cf3f95af2f4ee5fb1a20fface444233a18f41c4c13426c0" } diff --git a/.sqlx/query-6b6175ca191732b4ece42b2a5083cbbe2eaf29bd3d2b409070b60a8a9661ea53.json b/.sqlx/query-6b6175ca191732b4ece42b2a5083cbbe2eaf29bd3d2b409070b60a8a9661ea53.json new file mode 100644 index 0000000..3d532b0 --- /dev/null +++ b/.sqlx/query-6b6175ca191732b4ece42b2a5083cbbe2eaf29bd3d2b409070b60a8a9661ea53.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH j AS (\n SELECT\n (value ->> 'task_id')::text AS task_id,\n (value ->> 'attempt')::integer AS attempt,\n value -> 'result' AS result,\n value ->> 'status' AS status\n FROM jsonb_array_elements($1::jsonb) AS value\n),\nlocked AS (\n SELECT jobs.id\n FROM apalis.jobs AS jobs\n INNER JOIN j ON j.task_id = jobs.id\n WHERE jobs.lock_by = $2\n ORDER BY jobs.id\n FOR UPDATE\n)\nUPDATE apalis.jobs AS jobs\nSET\n status = j.status,\n attempts = j.attempt,\n last_result = j.result,\n done_at = NOW()\nFROM j\nINNER JOIN locked ON locked.id = j.task_id\nWHERE jobs.id = locked.id;\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6b6175ca191732b4ece42b2a5083cbbe2eaf29bd3d2b409070b60a8a9661ea53" +} diff --git a/.sqlx/query-6ce4a9f7891abb2452136bef3a9d9065d452490dc7dde0d6f0590838f386f39c.json b/.sqlx/query-6ce4a9f7891abb2452136bef3a9d9065d452490dc7dde0d6f0590838f386f39c.json deleted file mode 100644 index 679b8ee..0000000 --- a/.sqlx/query-6ce4a9f7891abb2452136bef3a9d9065d452490dc7dde0d6f0590838f386f39c.json +++ /dev/null @@ -1,185 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n apalis.jobs\nSET\n status = 'Running',\n lock_at = now(),\n lock_by = $2\nWHERE\n (\n status = 'Pending'\n OR status = 'Queued'\n OR (\n status = 'Failed'\n AND attempts < max_attempts\n )\n )\n AND run_at < now()\n AND id = ANY($1) RETURNING *;\n", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job", - "type_info": "Bytea", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "job" - } - } - }, - { - "ordinal": 1, - "name": "id", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "id" - } - } - }, - { - "ordinal": 2, - "name": "job_type", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "job_type" - } - } - }, - { - "ordinal": 3, - "name": "status", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "status" - } - } - }, - { - "ordinal": 4, - "name": "attempts", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "attempts" - } - } - }, - { - "ordinal": 5, - "name": "max_attempts", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "max_attempts" - } - } - }, - { - "ordinal": 6, - "name": "run_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "run_at" - } - } - }, - { - "ordinal": 7, - "name": "last_result", - "type_info": "Jsonb", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "last_result" - } - } - }, - { - "ordinal": 8, - "name": "lock_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "lock_at" - } - } - }, - { - "ordinal": 9, - "name": "lock_by", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "lock_by" - } - } - }, - { - "ordinal": 10, - "name": "done_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "done_at" - } - } - }, - { - "ordinal": 11, - "name": "priority", - "type_info": "Int4", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "priority" - } - } - }, - { - "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "metadata" - } - } - }, - { - "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", - "origin": { - "Table": { - "table": "apalis.jobs", - "name": "idempotency_key" - } - } - } - ], - "parameters": { - "Left": [ - "TextArray", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "6ce4a9f7891abb2452136bef3a9d9065d452490dc7dde0d6f0590838f386f39c" -} diff --git a/.sqlx/query-75a941c14fe29a7d1a4d64d70ca39dacfbc047c8051eafbd521a6a9d7845dfda.json b/.sqlx/query-75a941c14fe29a7d1a4d64d70ca39dacfbc047c8051eafbd521a6a9d7845dfda.json new file mode 100644 index 0000000..102964f --- /dev/null +++ b/.sqlx/query-75a941c14fe29a7d1a4d64d70ca39dacfbc047c8051eafbd521a6a9d7845dfda.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE apalis.workers w\nSET last_seen = NOW()\nWHERE\n w.id = $1\n AND w.worker_type = $2\n AND (\n SELECT COUNT(*)\n FROM apalis.jobs j\n WHERE j.lock_by = w.id\n AND j.id = ANY($3::text[])\n ) = cardinality($3::text[]);\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "75a941c14fe29a7d1a4d64d70ca39dacfbc047c8051eafbd521a6a9d7845dfda" +} diff --git a/.sqlx/query-97a3f2471a71ec938f9ed109bdeb293be2dbe386758ed6bb75cbf285555f6cec.json b/.sqlx/query-97a3f2471a71ec938f9ed109bdeb293be2dbe386758ed6bb75cbf285555f6cec.json new file mode 100644 index 0000000..18a1663 --- /dev/null +++ b/.sqlx/query-97a3f2471a71ec938f9ed109bdeb293be2dbe386758ed6bb75cbf285555f6cec.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH stale AS (\n SELECT\n jobs.id\n FROM\n apalis.jobs\n INNER JOIN apalis.workers ON jobs.lock_by = workers.id\n WHERE\n (\n jobs.status = 'Running'\n OR jobs.status = 'Queued'\n )\n AND NOW() - workers.last_seen >= $1\n AND workers.worker_type = $2 FOR\n UPDATE\n OF jobs SKIP LOCKED\n)\nUPDATE\n apalis.jobs\nSET\n status = 'Pending',\n done_at = NULL,\n lock_by = NULL,\n lock_at = NULL,\n attempts = attempts + 1,\n last_result = '{\"Err\": \"Re-enqueued due to worker heartbeat timeout.\"}'\nFROM\n stale\nWHERE\n apalis.jobs.id = stale.id;\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Interval", + "Text" + ] + }, + "nullable": [] + }, + "hash": "97a3f2471a71ec938f9ed109bdeb293be2dbe386758ed6bb75cbf285555f6cec" +} diff --git a/.sqlx/query-a0dcac1deb02eb19959aedcde6e8864a459891ff9b5447882bcd6a43856b91f4.json b/.sqlx/query-a0dcac1deb02eb19959aedcde6e8864a459891ff9b5447882bcd6a43856b91f4.json index 65ebc0a..dda0584 100644 --- a/.sqlx/query-a0dcac1deb02eb19959aedcde6e8864a459891ff9b5447882bcd6a43856b91f4.json +++ b/.sqlx/query-a0dcac1deb02eb19959aedcde6e8864a459891ff9b5447882bcd6a43856b91f4.json @@ -137,23 +137,28 @@ }, { "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", + "name": "idempotency_key", + "type_info": "Text", "origin": { "Table": { "table": "apalis.jobs", - "name": "metadata" + "name": "idempotency_key" } } }, { "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", + "name": "metadata", + "type_info": { + "Custom": { + "name": "hstore", + "kind": "Simple" + } + }, "origin": { "Table": { "table": "apalis.jobs", - "name": "idempotency_key" + "name": "metadata" } } } diff --git a/.sqlx/query-aec15451aa407010d95e6014b0e6b4a361a6cc85c26c56ea82cdd2c37a123dac.json b/.sqlx/query-aec15451aa407010d95e6014b0e6b4a361a6cc85c26c56ea82cdd2c37a123dac.json index cd6d983..46aee2a 100644 --- a/.sqlx/query-aec15451aa407010d95e6014b0e6b4a361a6cc85c26c56ea82cdd2c37a123dac.json +++ b/.sqlx/query-aec15451aa407010d95e6014b0e6b4a361a6cc85c26c56ea82cdd2c37a123dac.json @@ -137,23 +137,28 @@ }, { "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", + "name": "idempotency_key", + "type_info": "Text", "origin": { "Table": { "table": "apalis.jobs", - "name": "metadata" + "name": "idempotency_key" } } }, { "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", + "name": "metadata", + "type_info": { + "Custom": { + "name": "hstore", + "kind": "Simple" + } + }, "origin": { "Table": { "table": "apalis.jobs", - "name": "idempotency_key" + "name": "metadata" } } } diff --git a/.sqlx/query-c9c61fd5581d8d84900e8f2e663dfb8ef66fcc6e2e2d7511ce830cd4ed197457.json b/.sqlx/query-c9c61fd5581d8d84900e8f2e663dfb8ef66fcc6e2e2d7511ce830cd4ed197457.json new file mode 100644 index 0000000..73d99ea --- /dev/null +++ b/.sqlx/query-c9c61fd5581d8d84900e8f2e663dfb8ef66fcc6e2e2d7511ce830cd4ed197457.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE\n apalis.jobs\nSET\n status = 'Pending',\n done_at = NULL,\n lock_by = NULL,\n lock_at = NULL,\n attempts = attempts + 1,\n last_result = '{\"Err\": \"Re-enqueued due to worker shutdown\"}' :: jsonb\nFROM\n apalis.workers\nWHERE\n apalis.jobs.lock_by = apalis.workers.id\n AND (apalis.jobs.status = 'Queued' OR apalis.jobs.status = 'Running')\n AND apalis.workers.worker_type = $1\n AND apalis.workers.id = $2\n AND apalis.jobs.id = ANY($3);\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "c9c61fd5581d8d84900e8f2e663dfb8ef66fcc6e2e2d7511ce830cd4ed197457" +} diff --git a/.sqlx/query-fc3801ddf6016402eb1ab46da4e6d733992fd5b98269ad25c25dc9e9d7a1db0a.json b/.sqlx/query-fc3801ddf6016402eb1ab46da4e6d733992fd5b98269ad25c25dc9e9d7a1db0a.json index 69ee75d..257d3b3 100644 --- a/.sqlx/query-fc3801ddf6016402eb1ab46da4e6d733992fd5b98269ad25c25dc9e9d7a1db0a.json +++ b/.sqlx/query-fc3801ddf6016402eb1ab46da4e6d733992fd5b98269ad25c25dc9e9d7a1db0a.json @@ -77,14 +77,19 @@ }, { "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", + "name": "idempotency_key", + "type_info": "Text", "origin": "Expression" }, { "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", + "name": "metadata", + "type_info": { + "Custom": { + "name": "hstore", + "kind": "Simple" + } + }, "origin": "Expression" } ], diff --git a/.sqlx/query-fe90494e81b507098f8b8704e8d61500f5e81da8d38ffd1e03db0148eb889c78.json b/.sqlx/query-fe90494e81b507098f8b8704e8d61500f5e81da8d38ffd1e03db0148eb889c78.json index 5e91f18..b586890 100644 --- a/.sqlx/query-fe90494e81b507098f8b8704e8d61500f5e81da8d38ffd1e03db0148eb889c78.json +++ b/.sqlx/query-fe90494e81b507098f8b8704e8d61500f5e81da8d38ffd1e03db0148eb889c78.json @@ -137,23 +137,28 @@ }, { "ordinal": 12, - "name": "metadata", - "type_info": "Jsonb", + "name": "idempotency_key", + "type_info": "Text", "origin": { "Table": { "table": "apalis.jobs", - "name": "metadata" + "name": "idempotency_key" } } }, { "ordinal": 13, - "name": "idempotency_key", - "type_info": "Text", + "name": "metadata", + "type_info": { + "Custom": { + "name": "hstore", + "kind": "Simple" + } + }, "origin": { "Table": { "table": "apalis.jobs", - "name": "idempotency_key" + "name": "metadata" } } } diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ac3e1..f5c928c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog -## Unreleased +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.0.0-rc.9] - 2026-09-17 - fix: confine apalis's objects to the `apalis` schema (#86): - `generate_ulid` is now `apalis.generate_ulid` and no longer depends on `pgcrypto` — its random bytes come from core `gen_random_uuid()`. The sole caller (`apalis.push_job`) is repointed and the `public.generate_ulid` copy is dropped (via a new forward migration; existing migrations are not rewritten). diff --git a/Cargo.lock b/Cargo.lock index 4f31bf5..11c6e97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,35 +3,30 @@ version = 4 [[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.5" +name = "aho-corasick" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ - "libc", + "memchr", ] [[package]] -name = "anyhow" -version = "1.0.102" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "apalis" -version = "1.0.0-rc.9" +version = "1.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7780d1e7082500a4fdb463b0a6fc1c00e4012cd9b2af101c26fcabbb2f390f2c" +checksum = "cc96f8162b465147c3bbf0fe27b3803d468456823d1e2e5db17a3cc9206f1ccc" dependencies = [ "apalis-core", "futures-util", "pin-project", + "serde", "thiserror", "tower", "tracing", @@ -39,9 +34,9 @@ dependencies = [ [[package]] name = "apalis-codec" -version = "0.1.0-rc.9" +version = "0.1.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c506e7f00c7c9c38eeb02290b3ec6328695f0614a257faefaeb8e8286746a665" +checksum = "81f5cac07535fe1fd62ed316a2068c0efd886f41c2b1bf326f5d0d774fa523a3" dependencies = [ "apalis-core", "serde", @@ -50,31 +45,34 @@ dependencies = [ [[package]] name = "apalis-core" -version = "1.0.0-rc.9" +version = "1.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "797af42a40f6bc297365f2fed187b74d089c63641f57ce2a5e0f629db560cb47" +checksum = "736bb41e6475dafb9489fe67302f787381d5a511dbfa039cfe69e585bda6956a" dependencies = [ + "dashmap", "futures-channel", "futures-core", "futures-sink", "futures-timer", "futures-util", "pin-project", + "rand", "serde", "thiserror", "tower-layer", "tower-service", "tracing", + "ulid", + "uuid", ] [[package]] name = "apalis-postgres" -version = "1.0.0-rc.8" +version = "1.0.0-rc.9" dependencies = [ "apalis", "apalis-codec", "apalis-core", - "apalis-sql", "apalis-workflow", "async-std", "futures", @@ -86,37 +84,31 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", "ulid", ] -[[package]] -name = "apalis-sql" -version = "1.0.0-rc.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b555912820da093b004a055105af258df116edcea761db6b759d01aabac2ec" -dependencies = [ - "apalis-core", - "chrono", - "serde", - "serde_json", - "thiserror", - "time", -] - [[package]] name = "apalis-workflow" -version = "0.1.0-rc.9" +version = "0.1.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9392c07db462a5c0befec5d6685a37d9ec2424a244b7e7e021208c5c9544cef0" +checksum = "018253861e1cc1c9ec33d7b89bcfc60b38588d653aaab85555b122184cae5faa" dependencies = [ + "apalis-codec", "apalis-core", - "futures", + "dashmap", + "futures-channel", + "futures-core", + "futures-sink", + "futures-util", "petgraph", "serde", + "serde_json", "thiserror", "tower", "tracing", - "ulid", ] [[package]] @@ -206,7 +198,7 @@ version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "event-listener-strategy", "pin-project-lite", ] @@ -260,9 +252,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" @@ -272,9 +264,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -290,18 +282,18 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", ] [[package]] name = "blocking" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" dependencies = [ "async-channel 2.5.0", "async-task", @@ -312,9 +304,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -324,15 +316,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.61" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", "shlex", @@ -340,33 +332,19 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - -[[package]] -name = "chrono" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", + "cpufeatures 0.3.1", + "rand_core", ] [[package]] @@ -411,9 +389,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -435,24 +413,24 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -477,13 +455,17 @@ dependencies = [ ] [[package]] -name = "deranged" -version = "0.5.8" +name = "dashmap" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ - "powerfmt", - "serde_core", + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] @@ -493,7 +475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "crypto-common 0.1.7", + "crypto-common 0.1.6", ] [[package]] @@ -502,20 +484,20 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "crypto-common 0.2.2", "ctutils", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -526,9 +508,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.15.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" dependencies = [ "serde", ] @@ -567,11 +549,10 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -582,21 +563,21 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "pin-project-lite", ] [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fixedbitset" @@ -653,9 +634,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -668,9 +649,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -678,15 +659,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -706,9 +687,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -725,38 +706,38 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -771,9 +752,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -792,28 +773,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasip2", - "wasip3", + "r-efi", + "rand_core", ] [[package]] @@ -828,6 +795,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" @@ -850,15 +823,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ "hashbrown 0.16.1", ] @@ -871,9 +844,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -901,42 +874,18 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -948,9 +897,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -961,9 +910,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -975,16 +924,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -995,15 +945,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1014,12 +964,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "idna" version = "1.1.0" @@ -1043,14 +987,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.0", - "serde", - "serde_core", + "hashbrown 0.17.1", ] [[package]] @@ -1061,13 +1003,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1081,22 +1022,22 @@ dependencies = [ ] [[package]] -name = "leb128fmt" -version = "0.1.0" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "pkg-config", "vcpkg", @@ -1110,9 +1051,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1125,13 +1066,22 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" dependencies = [ "value-bag", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.11.0" @@ -1144,15 +1094,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mio" -version = "1.2.0" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", @@ -1177,10 +1127,13 @@ dependencies = [ ] [[package]] -name = "num-conv" -version = "0.2.1" +name = "nu-ansi-term" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] [[package]] name = "num-traits" @@ -1199,9 +1152,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl" -version = "0.10.79" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ "bitflags", "cfg-if", @@ -1219,7 +1172,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1230,9 +1183,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.115" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -1290,22 +1243,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1333,9 +1286,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polling" @@ -1353,62 +1306,31 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -1417,58 +1339,46 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.4" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "rand_chacha", - "rand_core 0.9.5", + "chacha20", + "getrandom 0.4.3", + "rand_core", ] [[package]] -name = "rand" +name = "rand_core" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core 0.10.1", -] +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "redox_syscall" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "bitflags", ] [[package]] -name = "rand_core" -version = "0.9.5" +name = "regex-automata" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ - "getrandom 0.3.4", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "rand_core" -version = "0.10.1" +name = "regex-syntax" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "ring" @@ -1499,9 +1409,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "once_cell", "ring", @@ -1513,18 +1423,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -1533,9 +1443,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "schannel" @@ -1575,17 +1485,11 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1593,29 +1497,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1640,10 +1544,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1662,15 +1572,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "slab" @@ -1680,18 +1599,18 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -1699,9 +1618,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -1731,11 +1650,10 @@ dependencies = [ "base64", "bytes", "cfg-if", - "chrono", "crc", "crossbeam-queue", "either", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-core", "futures-intrusive", "futures-io", @@ -1753,7 +1671,6 @@ dependencies = [ "sha2 0.10.9", "smallvec", "thiserror", - "time", "tokio", "tokio-stream", "toml", @@ -1772,7 +1689,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.119", ] [[package]] @@ -1796,7 +1713,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.119", "thiserror", "tokio", "url", @@ -1811,7 +1728,6 @@ dependencies = [ "bitflags", "byteorder", "bytes", - "chrono", "crc", "digest 0.11.3", "dotenvy", @@ -1826,7 +1742,6 @@ dependencies = [ "sha2 0.11.0", "sqlx-core", "thiserror", - "time", "tracing", ] @@ -1840,7 +1755,6 @@ dependencies = [ "base64", "bitflags", "byteorder", - "chrono", "crc", "dotenvy", "etcetera", @@ -1854,7 +1768,7 @@ dependencies = [ "log", "md-5", "memchr", - "rand 0.10.1", + "rand", "serde", "serde_json", "sha2 0.11.0", @@ -1862,7 +1776,6 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time", "tracing", "whoami", ] @@ -1874,7 +1787,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" dependencies = [ "atoi", - "chrono", "flume", "form_urlencoded", "futures-channel", @@ -1888,7 +1800,6 @@ dependencies = [ "serde", "sqlx-core", "thiserror", - "time", "tracing", "url", ] @@ -1918,9 +1829,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -1935,13 +1857,13 @@ checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -1951,7 +1873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -1959,60 +1881,38 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] -name = "time" -version = "0.3.47" +name = "thread_local" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", + "cfg-if", ] [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2020,24 +1920,15 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.52.2" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2050,20 +1941,20 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -2072,9 +1963,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", @@ -2173,7 +2064,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2183,22 +2074,53 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ulid" -version = "1.2.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +checksum = "947dde63b6d514cc5e044edad4e0ca7261afd1099d16c83d942cb2b2f348689c" dependencies = [ - "rand 0.9.4", + "rand", "serde", + "uuid", "web-time", ] @@ -2210,9 +2132,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "ab72a15cf68d77cb0987d3684aa8a45c5ef827e8cb49ee2f30bfd7ba2feb519f" [[package]] name = "unicode-normalization" @@ -2229,12 +2151,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "untrusted" version = "0.9.0" @@ -2259,11 +2175,30 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "sha1_smol", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "value-bag" -version = "1.12.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" +checksum = "2799ffb329a792ecfd902b71306c8a815a6ef1c0470fa9953a6aa4d4cecbe511" [[package]] name = "vcpkg" @@ -2283,29 +2218,11 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -2316,9 +2233,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -2326,9 +2243,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2336,60 +2253,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-time" version = "1.1.0" @@ -2402,53 +2285,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] [[package]] name = "whoami" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" [[package]] name = "windows-link" @@ -2456,24 +2304,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2565,111 +2395,17 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -2678,68 +2414,48 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -2748,9 +2464,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -2759,17 +2475,17 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 9d9a668..3548e1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-postgres" -version = "1.0.0-rc.8" +version = "1.0.0-rc.9" authors = ["Njuguna Mureithi "] edition = "2024" repository = "https://github.com/apalis-dev/apalis-postgres" @@ -14,32 +14,35 @@ categories = ["asynchronous", "database", "network-programming"] publish = true [features] -default = ["migrate", "tokio-comp", "chrono"] +default = ["migrate", "tokio-comp"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = ["async-std", "sqlx/runtime-async-std", "sqlx/tls-rustls"] -async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std", "sqlx/tls-native-tls"] +async-std-comp-native-tls = [ + "async-std", + "sqlx/runtime-async-std", + "sqlx/tls-native-tls", +] tokio-comp = ["tokio", "sqlx/runtime-tokio", "sqlx/tls-rustls"] tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio", "sqlx/tls-native-tls"] -chrono = ["apalis-sql/chrono", "sqlx/chrono"] -time = ["apalis-sql/time", "sqlx/time"] [dependencies] -apalis-core = { version = "1.0.0-rc.9", default-features = false, features = [ +apalis-core = { version = "1.0.0-rc.10", default-features = false, features = [ "sleep", + "ulid", ] } -apalis-sql = { version = "1.0.0-rc.9", default-features = false } -apalis-codec = { version = "0.1.0-rc.9", features = ["json"] } +apalis-codec = { version = "0.1.0-rc.10", features = ["json"] } serde = { version = "1", features = ["derive"], default-features = false } -pin-project = "1.1.10" +pin-project = "1.1.13" serde_json = "1" -futures = "0.3.30" +futures = "0.3.34" thiserror = "2" tokio = { version = "1", features = [ "rt", "net", ], optional = true, default-features = false } -async-std = { version = "1.13.0", optional = true, default-features = false } -ulid = { version = "1", features = ["serde"] } +async-std = { version = "1.13.2", optional = true, default-features = false } +ulid = { version = "3", features = ["serde"] } +tracing = "0.1" [dependencies.sqlx] @@ -49,7 +52,9 @@ features = ["postgres", "json", "sqlx-toml"] [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -once_cell = "1.19.0" -apalis = { version = "1.0.0-rc.9" } -apalis-workflow = { version = "0.1.0-rc.9" } -futures-util = "0.3.30" +once_cell = "1.21.4" +apalis = { version = "1.0.0-rc.10" } +apalis-workflow = { version = "0.1.0-rc.10" } +futures-util = "0.3.34" +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +tokio-util = "0.7.19" diff --git a/README.md b/README.md index e452f8b..42787a8 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,7 @@ Background task processing in rust using `apalis` and `postgres` ## Storage Types - [`PostgresStorage`]: Standard polling-based storage. -- [`PostgresStorageWithListener`]: Event-driven storage using Postgres `NOTIFY` for low-latency job fetching. -- [`SharedPostgresStorage`]: Shared storage for multiple job types, uses Postgres `NOTIFY`. +- [`PostgresStorageFactory`]: Shared storage for multiple job types, uses Postgres `NOTIFY`. The naming is designed to clearly indicate the storage mechanism and its capabilities, but under the hood the result is the `PostgresStorage` struct with different configurations. @@ -39,9 +38,9 @@ async fn main() { let mut start = 0usize; let mut items = stream::repeat_with(move || { start += 1; - let task = Task::builder(start) + let task = TaskBuilder::new(start) .run_after(Duration::from_secs(1)) - .with_ctx(PgContext::new().with_priority(1)) + .priority(2) .build(); task }) @@ -59,7 +58,10 @@ async fn main() { } ``` -### `NOTIFY` listener example +### Pubsub listener example + +Uses `LISTEN/NOTIFY` to subscribe to events. Each worker gets its own listener. To share a listener b +please use `PostgresStorageFactory` ```rust,no_run use std::time::Duration; @@ -73,33 +75,34 @@ async fn main() { let pool = PgPool::connect(env!("DATABASE_URL")).await.unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let lazy_strategy = StrategyBuilder::new() - .apply(IntervalStrategy::new(Duration::from_secs(5))) - .build(); - let config = Config::new("queue") - .with_poll_interval(lazy_strategy) - .set_buffer_size(5); - let backend = PostgresStorage::new_with_notify(&pool, &config); + let lazy_strategy = Strategy::new() + .interval(Duration::from_secs(5)); + let config = Config::default() + .queue("my-queue") + .batch_size(5); + let backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub() + .poll_with_strategy(lazy_strategy); tokio::spawn({ let pool = pool.clone(); - let config = config.clone(); async move { tokio::time::sleep(Duration::from_secs(2)).await; let mut start = 0; let items = stream::repeat_with(move || { start += 1; // Construct compact task - Task::builder(serde_json::to_vec(&start).unwrap()) - .with_ctx(PgContext::new().with_priority(start)) + TaskBuilder::new(serde_json::to_vec(&start).unwrap()) + .priority(start) .build() }) .take(20) .collect::>() .await; - // You can still use backend.push - // This example shows how to do it with just a pool - apalis_postgres::sink::push_tasks(&pool, config, items).await.unwrap(); + let mut tx = pool.begin().await.unwrap(); + apalis_postgres::queries::push_tasks(&mut *tx, "my-queue", items).await.unwrap(); + tx.commit().await.unwrap() } }); @@ -126,7 +129,7 @@ use futures::stream::{self, StreamExt}; #[tokio::main] async fn main() { - let workflow = Workflow::new("odd-numbers-workflow") + let workflow = SteppedFlow::new("odd-numbers-workflow") .and_then(|a: usize| async move { Ok::<_, BoxDynError>((0..=a).collect::>()) }) @@ -147,7 +150,8 @@ async fn main() { let pool = PgPool::connect(env!("DATABASE_URL")).await.unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let mut backend = PostgresStorage::new_with_config(&pool, &Config::new("test-workflow")); + let config = Config::default().queue("workflow"); + let mut backend = PostgresStorage::new(&pool).with_config(config); backend.push_start(100usize).await.unwrap(); @@ -174,7 +178,7 @@ This can improve performance if you have many types of jobs. use std::{collections::HashMap, time::Duration}; use apalis::prelude::*; -use apalis_postgres::{shared::SharedPostgresStorage, *}; +use apalis_postgres::{factory::PostgresStorageFactory, *}; use futures::stream; #[tokio::main] @@ -183,11 +187,11 @@ async fn main() { .await .unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let mut store = SharedPostgresStorage::new(pool); + let mut factory = PostgresStorageFactory::new(pool); - let mut map_store = store.make_shared().unwrap(); + let mut map_store = factory.create().unwrap(); - let mut int_store = store.make_shared().unwrap(); + let mut int_store = factory.create().unwrap(); map_store .push_stream(&mut stream::iter(vec![HashMap::::new()])) @@ -195,9 +199,9 @@ async fn main() { .unwrap(); int_store.push(99).await.unwrap(); - async fn send_reminder( + async fn send_reminder( _: T, - _task_id: TaskId, + _task_id: TaskId, wrk: WorkerContext, ) -> Result<(), BoxDynError> { tokio::time::sleep(Duration::from_secs(2)).await; @@ -222,38 +226,73 @@ Track your jobs using [apalis-board](https://github.com/apalis-dev/apalis-board) ## Upgrading to 1.0 -1.0 confines everything apalis creates to the `apalis` schema. Two things move out of `public`: +Starting with `1.0`, `apalis-postgres` keeps everything it creates inside the `apalis` PostgreSQL schema. + +This changes two things: + +- **Migration history** — SQLx migrations are now tracked in `apalis._sqlx_migrations` instead of `public._sqlx_migrations`. This prevents apalis-postgres's migration history from colliding with migrations belonging to your application. +- **`generate_ulid()`** — The function is now `apalis.generate_ulid()` and no longer requires the `pgcrypto` extension. Its random bytes are generated using PostgreSQL's built-in `gen_random_uuid()`. The old `public.generate_ulid()` function is removed. + +> **⚠️ Existing databases require a one-time migration.** +> If your database was created with a pre-`1.0` version of `apalis-postgres`, you must perform the migration below **before running any `1.0` migrations**. -- The sqlx migrations table is now tracked in `apalis._sqlx_migrations` (configured in `sqlx.toml`) instead of `public._sqlx_migrations`. This also keeps apalis's migration history from colliding with your own sqlx migrations on the same database. -- `generate_ulid()` is now `apalis.generate_ulid()` and no longer depends on the `pgcrypto` extension — its random bytes come from core `gen_random_uuid()`. The `public.generate_ulid()` copy is dropped. +### Existing databases — one-time migration -### Existing databases: one-time manual step +This applies regardless of how you run migrations: `PostgresStorage::setup()`, `sqlx-cli`, copied migration files, or a custom/merged `Migrator`. -This applies to **every** way of applying migrations — `PostgresStorage::setup()`, sqlx-cli, copied migration files, or a merged `Migrator`. Run this **once per database, before upgrading**: +Run this **once per database, before upgrading**: ```sql --- Move apalis's existing migration history into the apalis schema. +-- Move apalis-postgres' migration history into the apalis schema. ALTER TABLE public._sqlx_migrations SET SCHEMA apalis; --- The first migration gained `IF NOT EXISTS` (so the apalis schema can be --- created before the tracking table on fresh installs). Re-stamp its checksum --- so the migrator doesn't reject it as modified: +-- The first migration changed to use IF NOT EXISTS so that the apalis +-- schema can be created before the migration table on fresh installs. +-- Re-stamp its checksum to match the 1.0 migration. UPDATE apalis._sqlx_migrations - SET checksum = decode('d0839c6f57a379769dc27ccd581feb3d2709239c8f138e05271c9e3c760c4517a78a4d8912ab3d63b074b28d15ec74e9', 'hex') + SET checksum = decode( + 'd0839c6f57a379769dc27ccd581feb3d2709239c8f138e05271c9e3c760c4517a78a4d8912ab3d63b074b28d15ec74e9', + 'hex' + ) WHERE version = 20220530084123; ``` -Run it **before** upgrading. If you upgrade first without it, the migrator re-runs the first migration against your existing objects and fails with e.g. `function "notify_new_jobs" already exists`. If you've already hit that failure, an empty `apalis._sqlx_migrations` may have been created, which makes the `ALTER TABLE` above fail because the name is taken — drop it first: +> **❗ Do this before upgrading.** +> If you upgrade first, the migrator may not find the existing migration history and will attempt to re-run the first migration against objects that already exist, causing errors such as: +> +> ```log +> function "notify_new_jobs" already exists +> ``` + +#### If you already upgraded and the migration failed + +The `apalis._sqlx_migrations` table may have been created (empty) before the migration failed. Remove it first: ```sql DROP TABLE apalis._sqlx_migrations; ``` -then run the two statements above. +Then run the two statements from the [one-time migration](#existing-databases--one-time-migration) above. + +#### Custom `Migrator` + +If you maintain your own `Migrator` and merge in `PostgresStorage::migrations()`, your migration tracking table stays wherever your existing SQLx configuration puts it. + +> **ℹ️ Note:** Do not move your migration table in this case. Skip the `ALTER TABLE` statement and only update the checksum in your existing `_sqlx_migrations` table: + + +```sql +UPDATE ._sqlx_migrations + SET checksum = decode( + 'd0839c6f57a379769dc27ccd581feb3d2709239c8f138e05271c9e3c760c4517a78a4d8912ab3d63b074b28d15ec74e9', + 'hex' + ) + WHERE version = 20220530084123; +``` -If you maintain your **own** `Migrator` (merging in `PostgresStorage::migrations()`), your tracking table stays where it is — skip the `ALTER TABLE` and run only the `UPDATE`, targeting your table name. +### Fresh databases -Fresh databases need none of this — `sqlx.toml` creates the `apalis` schema and tracking table for you. +✅ No manual migration is required. On a fresh database, the `1.0` migrations automatically create the `apalis` schema and place apalis-postgres's migration history in `apalis._sqlx_migrations`. ### `pgcrypto` diff --git a/deny.toml b/deny.toml index 75e1cc7..cf4a2a0 100644 --- a/deny.toml +++ b/deny.toml @@ -35,9 +35,11 @@ skip-tree = [ { name = "getrandom" }, { name = "hashbrown" }, { name = "rand" }, - { name = "rand_chacha" }, + { name = "async-channel" }, { name = "rand_core" }, { name = "redox_syscall" }, + { name = "event-listener" }, + { name = "syn" }, # sqlx 0.9 pulls two incompatible digest stacks internally: sqlx-core uses # digest 0.10 / sha2 0.10, while sqlx-postgres (hmac/md-5/hkdf) uses 0.11. { name = "block-buffer" }, diff --git a/examples/basic.rs b/examples/basic.rs index b9fb8a7..2a82414 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -1,22 +1,32 @@ use std::time::Duration; -use apalis::{layers::retry::RetryPolicy, prelude::*}; -use apalis_postgres::*; -use apalis_sql::ext::TaskBuilderExt; +use apalis::prelude::*; +use apalis_postgres::{Config, *}; use futures::stream::{self, StreamExt}; +use sqlx::postgres::PgPoolOptions; #[tokio::main] async fn main() { let db = std::env::var("DATABASE_URL").unwrap(); - let pool = PgPool::connect(&db).await.unwrap(); + + // Configure the pool options and set max connections to 50 + let pool = PgPoolOptions::new().connect(&db).await.unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let mut backend = PostgresStorage::new(&pool); + let config = Config::default() + .queue("high-priority") + .batch_size(100) + .lock_tasks(false) + .heartbeat_interval(Duration::from_secs(1)) + .missed_heartbeats(10); + let mut backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub(); // Push some tasks as a stream let mut start = 0usize; let mut items = stream::repeat_with(move || { start += 1; - let task = Task::builder(start) + let task = TaskBuilder::new(start) .run_after(Duration::from_secs(1)) .priority(1) .max_attempts(5) @@ -26,18 +36,17 @@ async fn main() { .take(10); backend.push_all(&mut items).await.unwrap(); - async fn send_reminder(item: usize, _wrk: WorkerContext) -> Result<(), BoxDynError> { - if item.is_multiple_of(3) { - println!("Reminding about item: {} but failing", item); - return Err("Failed to send reminder".into()); + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + if item == 10 { + wrk.stop()?; } - println!("Reminding about item: {}", item); Ok(()) } - let worker = WorkerBuilder::new("worker-1") + let worker = WorkerBuilder::new("basic-worker") .backend(backend) - .retry(RetryPolicy::retries(1)) + .parallelize(tokio::spawn) .build(send_reminder); + worker.run().await.unwrap(); } diff --git a/examples/dag.rs b/examples/dag.rs index 434cf30..36bd103 100644 --- a/examples/dag.rs +++ b/examples/dag.rs @@ -1,5 +1,7 @@ +use std::time::Duration; + use apalis::prelude::*; -use apalis_postgres::*; +use apalis_postgres::{Config, *}; use apalis_workflow::*; async fn get_name(user_id: u32) -> Result { @@ -25,12 +27,12 @@ async fn collector( #[tokio::main] async fn main() { - let dag_flow = DagFlow::new("user-etl-workflow"); - let get_name = dag_flow.node(get_name); - let get_age = dag_flow.node(get_age); - let get_address = dag_flow.node(get_address); + let dag_flow = GraphFlow::new("user-etl-workflow"); + let get_name = dag_flow.add_task(get_name); + let get_age = dag_flow.add_task(get_age); + let get_address = dag_flow.add_task(get_address); dag_flow - .node(collector) + .add_task(collector) .depends_on((&get_name, &get_age, &get_address)); // Order and types matters here dag_flow.validate().unwrap(); @@ -41,9 +43,14 @@ async fn main() { .await .unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let mut backend = PostgresStorage::new_with_config(&pool, &Config::new("test-workflow")); - backend.push_start(vec![42u32, 43, 44]).await.unwrap(); + let config = Config::default().queue("test-workflow"); + let mut backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub() + .poll_with_interval(Duration::from_secs(1)); // Wake the worker once a second if its sleeping + + backend.start_fan_out(vec![42u32, 43, 44]).await.unwrap(); let worker = WorkerBuilder::new("rango-tango") .backend(backend) diff --git a/examples/pubsub.rs b/examples/pubsub.rs index d0f6aa8..e4640fa 100644 --- a/examples/pubsub.rs +++ b/examples/pubsub.rs @@ -1,52 +1,79 @@ -use std::time::Duration; +use std::time::{Duration, Instant}; use apalis::prelude::*; -use apalis_postgres::*; -use futures::stream::{self, StreamExt}; +use apalis_postgres::{Config, *}; +use sqlx::pool::PoolOptions; +use tracing::{Instrument, Level, info}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[tokio::main] async fn main() { - let pool = PgPool::connect(&std::env::var("DATABASE_URL").unwrap()) - .await + use tracing_subscriber::{EnvFilter, fmt}; + let fmt_layer = fmt::layer(); + let filter_layer = EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new("trace")) + .unwrap(); + + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .init(); + + let pool = PoolOptions::new() + .connect_lazy(&std::env::var("DATABASE_URL").unwrap()) .unwrap(); + PostgresStorage::setup(&pool).await.unwrap(); - let lazy_strategy = StrategyBuilder::new() - .apply(IntervalStrategy::new(Duration::from_secs(5))) - .build(); - let config = Config::new("queue") - .with_poll_interval(lazy_strategy) - .set_buffer_size(5); - let backend = PostgresStorage::new_with_notify(&pool, &config); + let queue = "queue"; + let config = Config::default().queue(queue).batch_size(10); + + let backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub() + .poll_with_interval(Duration::from_secs(10)) + .instrumented(tracing::span!(Level::INFO, "postgres-pubsub")); tokio::spawn({ let pool = pool.clone(); - let config = config.clone(); + async move { - tokio::time::sleep(Duration::from_secs(2)).await; + tokio::time::sleep(Duration::from_secs(3)).await; + let mut conn = pool.acquire().await.unwrap().detach(); let mut start = 0; - let items = stream::repeat_with(move || { + while start < 100 { + tokio::time::sleep(Duration::from_secs(1)).await; start += 1; - Task::builder(serde_json::to_vec(&start).unwrap()) - .run_after(Duration::from_secs(1)) - .with_ctx(PgContext::new().with_priority(start)) - .build() - }) - .take(20) - .collect::>() - .await; - apalis_postgres::sink::push_tasks(&pool, config, items) - .await - .unwrap(); + let tasks = TaskBuilder::new(serde_json::to_vec(&start).unwrap()) + .priority(start) + .build(); + + apalis_postgres::queries::push_tasks(&mut conn, queue, vec![tasks]) + .await + .unwrap(); + } } }); - async fn send_reminder(_item: usize, _wrk: WorkerContext) -> Result<(), BoxDynError> { + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + info!("Found Item: {item}"); + if item == 10 { + wrk.stop()?; + } Ok(()) } + let start = Instant::now(); let worker = WorkerBuilder::new("worker-2") .backend(backend) + .enable_tracing() + .on_event(|_, e| info!("{:?}", e)) .build(send_reminder); - worker.run().await.unwrap(); + worker + .run() + .instrument(tracing::span!(Level::INFO, "worker-2")) + .await + .unwrap(); + + info!("Elapsed: {:?}", start.elapsed()); } diff --git a/examples/shared.rs b/examples/shared.rs index 9367e87..1530390 100644 --- a/examples/shared.rs +++ b/examples/shared.rs @@ -1,29 +1,51 @@ use std::{collections::HashMap, time::Duration}; use apalis::prelude::*; -use apalis_postgres::{shared::SharedPostgresStorage, *}; -use futures::stream; +use apalis_postgres::{Config, factory::PostgresStorageFactory, *}; +use futures::{ + FutureExt, StreamExt, TryStreamExt, + stream::{self, FuturesUnordered}, +}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[tokio::main] async fn main() { + use tracing_subscriber::{EnvFilter, fmt}; + let fmt_layer = fmt::layer(); + let filter_layer = EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new("debug")) + .unwrap(); + + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .init(); + let pool = PgPool::connect(&std::env::var("DATABASE_URL").unwrap()) .await .unwrap(); - let mut store = SharedPostgresStorage::new(pool); - let mut map_store = store.make_shared().unwrap(); + let config = Config::default() + .queue("int-store") + .batch_size(1) + .heartbeat_interval(Duration::from_secs(1)) + .missed_heartbeats(10); + let mut store = PostgresStorageFactory::new(pool); + + let mut map_store = store.create().unwrap(); - let mut int_store = store.make_shared().unwrap(); + let mut int_store = store.create_with_config(config).unwrap(); map_store .push_stream(&mut stream::iter(vec![HashMap::::new()])) .await .unwrap(); - int_store.push(99).await.unwrap(); - - async fn send_reminder( + let range = 0..10; + let mut stream = stream::iter(range).map(|i| i); + int_store.push_stream(&mut stream).await.unwrap(); + async fn send_reminder( _: T, - _task_id: TaskId, + _task_id: TaskId, wrk: WorkerContext, ) -> Result<(), BoxDynError> { tokio::time::sleep(Duration::from_secs(2)).await; @@ -31,11 +53,17 @@ async fn main() { Ok(()) } - let int_worker = WorkerBuilder::new("rango-tango-2") - .backend(int_store) - .build(send_reminder); + let workers = FuturesUnordered::new(); + for i in 0..5 { + let worker = WorkerBuilder::new(format!("worker-{}", i)) + .backend(int_store.clone()) + .build(send_reminder); + workers.push(worker.run().boxed()); + } let map_worker = WorkerBuilder::new("rango-tango-1") .backend(map_store) .build(send_reminder); - tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); + workers.push(map_worker.run().boxed()); + + workers.try_collect::>().await.unwrap(); } diff --git a/examples/stepped.rs b/examples/stepped.rs index 3e4ac11..c64d009 100644 --- a/examples/stepped.rs +++ b/examples/stepped.rs @@ -1,19 +1,20 @@ use std::time::Duration; use apalis::prelude::*; -use apalis_postgres::*; -use apalis_workflow::*; +use apalis_postgres::{Config, PgPool, PgTaskId, PostgresStorage}; +use apalis_workflow::SteppedFlow; #[tokio::main] async fn main() { - let workflow = Workflow::new("odd-numbers-workflow") + let workflow = SteppedFlow::new("odd-numbers-workflow") .and_then(|a: usize| async move { Ok::<_, BoxDynError>((0..=a).collect::>()) }) + .delay_for(Duration::from_secs(5)) .filter_map(|x| async move { if x % 2 != 0 { Some(x) } else { None } }) .delay_for(Duration::from_millis(1000)) .and_then( - |a: Vec, ctx: WorkerContext, task_id: PgTaskId| async move { + |a: Vec, wrk: WorkerContext, task_id: PgTaskId| async move { println!("Sum: {}", a.iter().sum::()); - ctx.stop().unwrap(); + wrk.stop().unwrap(); println!("Completed Task ID: {}", task_id); Ok::<(), BoxDynError>(()) }, @@ -23,9 +24,13 @@ async fn main() { .await .unwrap(); PostgresStorage::setup(&pool).await.unwrap(); - let mut backend = PostgresStorage::new_with_config(&pool, &Config::new("test-workflow")); + let config = Config::default().queue("test-workflow"); + let mut backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub() + .poll_with_interval(Duration::from_secs(1)); - backend.push_start(100usize).await.unwrap(); + backend.push(10usize).await.unwrap(); let worker = WorkerBuilder::new("rango-tango") .backend(backend) diff --git a/examples/unique_jobs.rs b/examples/unique_jobs.rs index 837f08b..8b46eab 100644 --- a/examples/unique_jobs.rs +++ b/examples/unique_jobs.rs @@ -12,13 +12,9 @@ async fn main() { PostgresStorage::setup(&pool).await.unwrap(); let mut backend = PostgresStorage::new(&pool); - let task_1 = TaskBuilder::new(42) - .with_idempotency_key(dedupe_key) - .build(); + let task_1 = TaskBuilder::new(42).idempotency_key(dedupe_key).build(); - let task_2 = TaskBuilder::new(43) - .with_idempotency_key(dedupe_key) - .build(); + let task_2 = TaskBuilder::new(43).idempotency_key(dedupe_key).build(); backend.push_task(task_1).await.unwrap(); backend.push_task(task_2).await.unwrap(); diff --git a/migrations/20260824233636_streamline_indices.sql b/migrations/20260824233636_streamline_indices.sql new file mode 100644 index 0000000..53fec15 --- /dev/null +++ b/migrations/20260824233636_streamline_indices.sql @@ -0,0 +1,26 @@ +DROP INDEX IF EXISTS apalis.TIdx; + +DROP INDEX IF EXISTS apalis.SIdx; + +DROP INDEX IF EXISTS apalis.JTIdx; + +DROP INDEX IF EXISTS apalis.Idx; + +CREATE INDEX IF NOT EXISTS idx_apalis_jobs_fetch_partial ON apalis.jobs ( + job_type, + priority DESC, + run_at ASC +) +WHERE + status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ); + +CREATE INDEX IF NOT EXISTS idx_apalis_jobs_lock_by ON apalis.jobs (lock_by); + +-- Keeps worker heartbeats fast +CREATE INDEX IF NOT EXISTS idx_apalis_workers_last_seen ON apalis.workers (last_seen); + +CREATE INDEX IF NOT EXISTS idx_apalis_workers_worker_type ON apalis.workers (worker_type); diff --git a/migrations/20260826101335_metadata_hstore.sql b/migrations/20260826101335_metadata_hstore.sql new file mode 100644 index 0000000..c9c5bb0 --- /dev/null +++ b/migrations/20260826101335_metadata_hstore.sql @@ -0,0 +1,22 @@ +CREATE EXTENSION IF NOT EXISTS hstore; + +ALTER TABLE + apalis.jobs +ADD + COLUMN metadata_hstore hstore; + +UPDATE + apalis.jobs +SET + metadata_hstore = ( + SELECT + hstore(array_agg(key), array_agg(value)) + FROM + jsonb_each_text(metadata) + ); + +ALTER TABLE + apalis.jobs DROP COLUMN metadata; + +ALTER TABLE + apalis.jobs RENAME COLUMN metadata_hstore TO metadata; diff --git a/migrations/20260902075949_remove_pl_in_get_jobs.sql b/migrations/20260902075949_remove_pl_in_get_jobs.sql new file mode 100644 index 0000000..9c838d1 --- /dev/null +++ b/migrations/20260902075949_remove_pl_in_get_jobs.sql @@ -0,0 +1,35 @@ +CREATE OR REPLACE FUNCTION apalis.get_jobs( + worker_id TEXT, + v_job_type TEXT, + v_job_count INTEGER DEFAULT 5 +) +RETURNS SETOF apalis.jobs +LANGUAGE sql +VOLATILE +AS $$ + WITH jobs AS ( + SELECT id + FROM apalis.jobs + WHERE + ( + status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ) + ) + AND run_at < now() + AND job_type = v_job_type + ORDER BY priority DESC, run_at ASC + LIMIT v_job_count + FOR UPDATE SKIP LOCKED + ) + UPDATE apalis.jobs j + SET + status = 'Queued', + lock_by = worker_id, + lock_at = now() + FROM jobs + WHERE j.id = jobs.id + RETURNING j.*; +$$; diff --git a/queries/backend/fetch_completed_tasks.sql b/queries/backend/fetch_completed_tasks.sql index c9727fb..da4a46a 100644 --- a/queries/backend/fetch_completed_tasks.sql +++ b/queries/backend/fetch_completed_tasks.sql @@ -1,6 +1,7 @@ SELECT id, status, + attempts as attempt, last_result AS result FROM apalis.jobs diff --git a/queries/backend/fetch_next.sql b/queries/backend/fetch_next.sql deleted file mode 100644 index f2552f9..0000000 --- a/queries/backend/fetch_next.sql +++ /dev/null @@ -1,20 +0,0 @@ -UPDATE Jobs -SET - status = 'Queued', - lock_by = ?1, - lock_at = strftime('%s', 'now') -WHERE - ROWID IN ( - SELECT ROWID - FROM Jobs - WHERE job_type = ?2 - AND ( - (status = 'Pending' AND lock_by IS NULL) - OR - (status = 'Failed' AND attempts < max_attempts) - ) - AND (run_at IS NULL OR run_at <= strftime('%s', 'now')) - ORDER BY priority DESC, run_at ASC, id ASC - LIMIT ?3 - ) -RETURNING * diff --git a/queries/backend/fetch_next_shared.sql b/queries/backend/fetch_next_shared.sql deleted file mode 100644 index 80fd356..0000000 --- a/queries/backend/fetch_next_shared.sql +++ /dev/null @@ -1,26 +0,0 @@ -UPDATE Jobs -SET - status = 'Queued', - lock_at = strftime('%s', 'now') -WHERE ROWID IN ( - SELECT ROWID - FROM Jobs - WHERE job_type IN ( - SELECT value FROM json_each(?1) - ) - AND status = 'Pending' - AND lock_by IS NULL - AND ( - run_at IS NULL - OR run_at <= strftime('%s', 'now') - ) - AND ROWID IN ( - SELECT value FROM json_each(?2) - ) - ORDER BY - priority DESC, - run_at ASC, - id ASC - LIMIT ?3 -) -RETURNING *; diff --git a/queries/backend/keep_alive.sql b/queries/backend/keep_alive.sql index 820b82a..c581742 100644 --- a/queries/backend/keep_alive.sql +++ b/queries/backend/keep_alive.sql @@ -1,6 +1,11 @@ -UPDATE - apalis.workers -SET - last_seen = NOW() +UPDATE apalis.workers w +SET last_seen = NOW() WHERE - id = $1 AND worker_type = $2; + w.id = $1 + AND w.worker_type = $2 + AND ( + SELECT COUNT(*) + FROM apalis.jobs j + WHERE j.lock_by = w.id + AND j.id = ANY($3::text[]) + ) = cardinality($3::text[]); diff --git a/queries/backend/reenqueue_orphaned.sql b/queries/backend/reenqueue_orphaned.sql index f00c6a1..505fc24 100644 --- a/queries/backend/reenqueue_orphaned.sql +++ b/queries/backend/reenqueue_orphaned.sql @@ -1,3 +1,19 @@ +WITH stale AS ( + SELECT + jobs.id + FROM + apalis.jobs + INNER JOIN apalis.workers ON jobs.lock_by = workers.id + WHERE + ( + jobs.status = 'Running' + OR jobs.status = 'Queued' + ) + AND NOW() - workers.last_seen >= $1 + AND workers.worker_type = $2 FOR + UPDATE + OF jobs SKIP LOCKED +) UPDATE apalis.jobs SET @@ -7,18 +23,7 @@ SET lock_at = NULL, attempts = attempts + 1, last_result = '{"Err": "Re-enqueued due to worker heartbeat timeout."}' +FROM + stale WHERE - id IN ( - SELECT - jobs.id - FROM - apalis.jobs - INNER JOIN apalis.workers ON lock_by = workers.id - WHERE - ( - status = 'Running' - OR status = 'Queued' - ) - AND NOW() - apalis.workers.last_seen >= $1 - AND apalis.workers.worker_type = $2 - ); + apalis.jobs.id = stale.id; diff --git a/queries/task/ack.sql b/queries/task/ack.sql deleted file mode 100644 index 869e6ba..0000000 --- a/queries/task/ack.sql +++ /dev/null @@ -1,10 +0,0 @@ -UPDATE - apalis.jobs -SET - status = $4, - attempts = $2, - last_result = $3, - done_at = NOW() -WHERE - id = $1 - AND lock_by = $5 diff --git a/queries/task/handle_result.sql b/queries/task/handle_result.sql new file mode 100644 index 0000000..24bf3b8 --- /dev/null +++ b/queries/task/handle_result.sql @@ -0,0 +1,25 @@ +WITH j AS ( + SELECT + (value ->> 'task_id')::text AS task_id, + (value ->> 'attempt')::integer AS attempt, + value -> 'result' AS result, + value ->> 'status' AS status + FROM jsonb_array_elements($1::jsonb) AS value +), +locked AS ( + SELECT jobs.id + FROM apalis.jobs AS jobs + INNER JOIN j ON j.task_id = jobs.id + WHERE jobs.lock_by = $2 + ORDER BY jobs.id + FOR UPDATE +) +UPDATE apalis.jobs AS jobs +SET + status = j.status, + attempts = j.attempt, + last_result = j.result, + done_at = NOW() +FROM j +INNER JOIN locked ON locked.id = j.task_id +WHERE jobs.id = locked.id; diff --git a/queries/task/lock_by_id.sql b/queries/task/lock_by_id.sql index 0315032..c5069a2 100644 --- a/queries/task/lock_by_id.sql +++ b/queries/task/lock_by_id.sql @@ -14,4 +14,4 @@ WHERE ) ) AND run_at < now() - AND id = ANY($1) RETURNING *; + AND id = ANY($1); diff --git a/queries/task/queue_by_id.sql b/queries/task/queue_by_id.sql index 813c2cc..ea22071 100644 --- a/queries/task/queue_by_id.sql +++ b/queries/task/queue_by_id.sql @@ -1,8 +1,7 @@ UPDATE apalis.jobs SET status = 'Queued', - lock_at = now(), - lock_by = $2 + lock_at = now() WHERE status = 'Pending' AND run_at < now() diff --git a/queries/task/sink.sql b/queries/task/sink.sql index 917f83b..51fa8e0 100644 --- a/queries/task/sink.sql +++ b/queries/task/sink.sql @@ -1,24 +1,23 @@ -INSERT INTO - apalis.jobs ( - id, - job_type, - job, - status, - attempts, - max_attempts, - run_at, - priority, - metadata, - idempotency_key - ) +INSERT INTO apalis.jobs ( + id, + job_type, + job, + status, + attempts, + max_attempts, + run_at, + priority, + metadata, + idempotency_key +) SELECT - unnest($1::text[]) as id, - $2::text as job_type, - unnest($3::bytea[]) as job, - 'Pending' as status, - 0 as attempts, - unnest($4::integer []) as max_attempts, - unnest($5::timestamptz []) as run_at, - unnest($6::integer []) as priority, - unnest($7::jsonb []) as metadata, - unnest($8::text []) as idempotency_key + unnest($1::text[]) AS id, + $2::text AS job_type, + unnest($3::bytea[]) AS job, + 'Pending' AS status, + 0 AS attempts, + unnest($4::integer[]) AS max_attempts, + to_timestamp(unnest($5::bigint[])) AS run_at, + unnest($6::integer[]) AS priority, + unnest($7::hstore[]) AS metadata, + unnest($8::text[]) AS idempotency_key diff --git a/queries/worker/reenqueue_abandoned.sql b/queries/worker/reenqueue_abandoned.sql new file mode 100644 index 0000000..627b18d --- /dev/null +++ b/queries/worker/reenqueue_abandoned.sql @@ -0,0 +1,17 @@ +UPDATE + apalis.jobs +SET + status = 'Pending', + done_at = NULL, + lock_by = NULL, + lock_at = NULL, + attempts = attempts + 1, + last_result = '{"Err": "Re-enqueued due to worker shutdown"}' :: jsonb +FROM + apalis.workers +WHERE + apalis.jobs.lock_by = apalis.workers.id + AND (apalis.jobs.status = 'Queued' OR apalis.jobs.status = 'Running') + AND apalis.workers.worker_type = $1 + AND apalis.workers.id = $2 + AND apalis.jobs.id = ANY($3); diff --git a/sqlx.toml b/sqlx.toml index 5554716..d25e778 100644 --- a/sqlx.toml +++ b/sqlx.toml @@ -2,3 +2,5 @@ create-schemas = ["apalis"] table-name = "apalis._sqlx_migrations" +[macros.type-overrides] +'TIMESTAMPTZ' = 'crate::timestamp::Timestamp' diff --git a/src/ack.rs b/src/ack.rs deleted file mode 100644 index 1d5bcec..0000000 --- a/src/ack.rs +++ /dev/null @@ -1,159 +0,0 @@ -use apalis_core::{ - error::AbortError, - error::BoxDynError, - layers::{Layer, Service}, - task::{Parts, status::Status}, - worker::{context::WorkerContext, ext::ack::Acknowledge}, -}; -use futures::{FutureExt, future::BoxFuture}; -use serde::Serialize; -use sqlx::PgPool; -use ulid::Ulid; - -use crate::{PgContext, PgTask}; - -#[derive(Debug, Clone)] -pub struct PgAck { - pool: PgPool, -} -impl PgAck { - pub fn new(pool: PgPool) -> Self { - Self { pool } - } -} - -impl Acknowledge for PgAck { - type Error = sqlx::Error; - type Future = BoxFuture<'static, Result<(), Self::Error>>; - fn ack( - &mut self, - res: &Result, - parts: &Parts, - ) -> Self::Future { - let task_id = parts.task_id; - let worker_id = parts.ctx.lock_by().clone(); - - let response = serde_json::to_value(res.as_ref().map_err(|e| e.to_string())); - let status = calculate_status(parts, res); - let attempt = parts.attempt.current() as i32; - let pool = self.pool.clone(); - async move { - let res = sqlx::query_file!( - "queries/task/ack.sql", - task_id - .ok_or(sqlx::Error::ColumnNotFound("TASK_ID_FOR_ACK".to_owned()))? - .to_string(), - attempt, - &response.map_err(|e| sqlx::Error::Decode(e.into()))?, - status.to_string(), - worker_id.ok_or(sqlx::Error::ColumnNotFound("WORKER_ID_LOCK_BY".to_owned()))? - ) - .execute(&pool) - .await?; - - if res.rows_affected() == 0 { - return Err(sqlx::Error::RowNotFound); - } - Ok(()) - } - .boxed() - } -} - -pub fn calculate_status( - parts: &Parts, - res: &Result, -) -> Status { - match &res { - Ok(_) => Status::Done, - Err(e) => match &e { - // Error::Abort(_) => State::Killed, - _ if parts.ctx.max_attempts() as usize <= parts.attempt.current() => Status::Killed, - _ => Status::Failed, - }, - } -} - -pub async fn lock_task(pool: &PgPool, task_id: &Ulid, worker_id: &str) -> Result<(), sqlx::Error> { - let task_id = vec![task_id.to_string()]; - sqlx::query_file!("queries/task/lock_by_id.sql", &task_id, &worker_id,) - .fetch_one(pool) - .await?; - Ok(()) -} - -#[derive(Debug, Clone)] - -pub struct LockTaskLayer { - pool: PgPool, -} - -impl LockTaskLayer { - pub fn new(pool: PgPool) -> Self { - Self { pool } - } -} - -impl Layer for LockTaskLayer { - type Service = LockTaskService; - - fn layer(&self, inner: S) -> Self::Service { - LockTaskService { - inner, - pool: self.pool.clone(), - } - } -} - -#[derive(Debug, Clone)] -pub struct LockTaskService { - inner: S, - pool: PgPool, -} - -impl Service> for LockTaskService -where - S: Service> + Send + 'static, - S::Future: Send + 'static, - S::Error: Into, - Args: Send + 'static, -{ - type Response = S::Response; - type Error = BoxDynError; - type Future = BoxFuture<'static, Result>; - - fn poll_ready( - &mut self, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.inner.poll_ready(cx).map_err(|e| e.into()) - } - - fn call(&mut self, req: PgTask) -> Self::Future { - let pool = self.pool.clone(); - let worker_id = req - .parts - .data - .get::() - .map(|w| w.name().to_owned()) - .unwrap(); - let parts = &req.parts; - let task_id = match &parts.task_id { - Some(id) => *id.inner(), - None => { - return async { - Err(sqlx::Error::ColumnNotFound("TASK_ID_FOR_LOCK".to_owned()).into()) - } - .boxed(); - } - }; - let fut = self.inner.call(req); - async move { - lock_task(&pool, &task_id, &worker_id) - .await - .map_err(AbortError::new)?; - fut.await.map_err(|e| e.into()) - } - .boxed() - } -} diff --git a/src/backend.rs b/src/backend.rs new file mode 100644 index 0000000..f64b56f --- /dev/null +++ b/src/backend.rs @@ -0,0 +1,221 @@ +use std::{ + marker::PhantomData, + task::{Context, Poll}, +}; + +use apalis_codec::json::JsonCodec; +use apalis_core::{ + backend::{ + Backend, BackendConfig, WireFormatBackend, + ext::poll_strategy::{PollWith, StreamStrategy}, + finalize::Durable, + persistence::{Persisted, TaskPersistLayer}, + }, + features_table, + worker::context::WorkerContext, +}; +use serde_json::Value; +use sqlx::PgPool; +use ulid::Ulid; + +use crate::{PgTask, config::Config, error::Error, persistence::SqlxPersistence, pubsub::Pubsub}; + +/// A backend for persisting and consuming jobs behind a postgres database +#[doc = features_table! { + setup = r#" + # { + # use apalis_postgres::PostgresStorage; + # use sqlx::PgPool; + # let pool = PgPool::connect(std::env::var("DATABASE_URL").unwrap().as_str()).await.unwrap(); + # PostgresStorage::setup(&pool).await.unwrap(); + # PostgresStorage::::new(&pool) + # }; + "#, + + Backend => supported("Supports storage and retrieval of tasks", true), + TaskSink => supported("Ability to push new tasks", true), + Serialization => supported("Serialization support for arguments", true), + Workflow => supported("Flexible enough to support workflows", true), + WebUI => supported("Expose a web interface for monitoring tasks", true), + FetchById => supported("Allow fetching a task by its ID", false), + RegisterWorker => supported("Allow registering a worker with the backend", false), + MakeShared => supported("Share one connection across multiple workers via [`PostgresStorageFactory`]", false), + WaitForCompletion => supported("Wait for tasks to complete without blocking", true), + ResumeById => supported("Resume a task by its ID", false), + ResumeAbandoned => supported("Resume abandoned tasks", false), + ListWorkers => supported("List all workers registered with the backend", false), + ListTasks => supported("List all tasks in the backend", false), +}] +/// +/// [`PostgresStorageFactory`]: crate::factory::PostgresStorageFactory +#[pin_project::pin_project] +pub struct PostgresStorage { + #[pin] + pub(crate) persistence: Persisted, + codec: JsonCodec, + _marker: PhantomData, +} + +impl Clone for PostgresStorage { + fn clone(&self) -> Self { + Self { + persistence: self.persistence.clone(), + codec: self.codec.clone(), + _marker: PhantomData, + } + } +} + +impl PostgresStorage<()> { + /// Runs the PostgreSQL storage migrations. + /// + /// ## Fresh databases + /// + /// No manual setup is required. Calling `setup()` will create the required + /// tables and migration history. + /// + /// ## Upgrading to `1.0` + /// + /// > **⚠️ Important:** Existing databases created by a pre-`1.0` version + /// > require a **one-time manual migration** before calling `setup()`. + /// + /// The `1.0` migration history is no longer relocated automatically by + /// `setup()`. Follow the **"Upgrading to 1.0"** section in the README to + /// perform the required transition. + /// + /// After the transition has been completed, `setup()` can be used normally + /// for subsequent migrations. + /// + /// ## Example + /// + /// ```no_run + /// use apalis_postgres::PostgresStorage; + /// use sqlx::PgPool; + /// + /// # async fn run(pool: PgPool) -> Result<(), apalis_postgres::Error> { + /// PostgresStorage::<()>::setup(&pool).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## Errors + /// + /// Returns an error if the migrations cannot be applied to the database. + #[cfg(feature = "migrate")] + pub async fn setup(pool: &PgPool) -> Result<(), Error> { + Self::migrations() + .run(pool) + .await + .map_err(sqlx::Error::from)?; + Ok(()) + } + + /// Get postgres migrations without running them + #[cfg(feature = "migrate")] + pub fn migrations() -> sqlx::migrate::Migrator { + sqlx::migrate!("./migrations") + } +} + +impl PostgresStorage { + /// Creates a new PostgresStorage instance. + pub fn new(pool: &PgPool) -> Self { + let config = Config::default().queue(std::any::type_name::()); + let persistence = Persisted::new(SqlxPersistence { + config, + pool: pool.clone(), + }); + Self { + _marker: PhantomData, + codec: JsonCodec::default(), + persistence, + } + } + + /// Mount a standalone [`Pubsub`] which uses its own connection under the hood + pub fn with_pubsub(self) -> PollWith> { + let pool = self.pool().clone(); + let config = self.config(); + let namespace = config.queue.to_string(); + PollWith::new(self, StreamStrategy::new(Pubsub::new(pool, namespace))) + } + + /// Configure a new PostgresStorage instance. + pub fn with_config(mut self, config: Config) -> Self { + self.persistence.config = config; + self + } + + /// Returns a reference to the pool. + pub fn pool(&self) -> &PgPool { + &self.persistence.pool + } + + /// Returns a reference to the config. + pub fn config(&self) -> &Config { + &self.persistence.config + } +} + +impl Backend for PostgresStorage { + type Task = PgTask; + type Error = Error; + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.persistence + .poll_ready(cx, worker, self.config().heartbeat_interval) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll>> { + self.persistence.poll_next(cx, worker) + } + + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.persistence.poll_close(cx, worker) + } +} + +impl BackendConfig for PostgresStorage { + type Args = Args; + + type Kind = Durable; + + type Id = Ulid; + + type Config = Config; + + type Layer = TaskPersistLayer, Value>; + + fn config(&self) -> &Self::Config { + &self.persistence.config + } + + fn middleware(&mut self, _: &mut WorkerContext) -> Self::Layer { + self.persistence + .layer(JsonCodec::::default(), self.config().batch_size) + .persist_results(self.config().persist_results) + .lock_tasks(self.config().lock_tasks) + } +} + +impl WireFormatBackend for PostgresStorage { + type Codec = JsonCodec; + + type Compact = Vec; + + fn codec(&self) -> &Self::Codec { + &self.codec + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..6bda667 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,247 @@ +use std::time::Duration; + +use apalis_core::backend::queue::Queue; +use serde::{Deserialize, Serialize}; + +/// Configuration for a worker's queue, batching, and liveness detection. +/// +/// `Config` controls how jobs are fetched from a queue and how worker +/// liveness is monitored. +/// +/// # Defaults +/// +/// - `batch_size`: `10` +/// - `heartbeat_interval`: `30` seconds +/// - `missed_heartbeats`: `2` +/// - `queue`: `"default"` +/// - `database_url`: `None` +/// - `lock_tasks`: `true` +/// - `persist_results`: `true` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Config { + /// The maximum number of jobs fetched in a single batch. + /// + /// Must be greater than zero. + #[serde(default = "default_batch_size")] + pub batch_size: usize, + + /// The interval between worker heartbeats. + #[serde(default = "default_heartbeat_interval")] + pub heartbeat_interval: Duration, + + /// The number of missed heartbeats allowed before a worker is + /// considered dead. + #[serde(default = "default_missed_heartbeats")] + pub missed_heartbeats: usize, + + /// The queue from which jobs are consumed. + pub queue: Queue, + + /// An optional database URL used by the worker. + pub database_url: Option, + + /// Whether tasks should be locked while being processed. + #[serde(default = "default_events")] + pub lock_tasks: bool, + + /// Whether job results should be persisted. + #[serde(default = "default_events")] + pub persist_results: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + batch_size: 10, + heartbeat_interval: Duration::from_secs(30), + missed_heartbeats: 10, + queue: Queue::from("default"), + database_url: None, + lock_tasks: true, + persist_results: true, + } + } +} + +fn default_batch_size() -> usize { + 10 +} + +fn default_heartbeat_interval() -> Duration { + Duration::from_secs(30) +} + +fn default_missed_heartbeats() -> usize { + 2 +} + +fn default_events() -> bool { + true +} + +impl Config { + /// Sets the maximum number of jobs to fetch in a single batch. + /// + /// Larger batches can improve throughput by reducing the number of + /// queue operations, while smaller batches can reduce memory usage + /// and improve job distribution between workers. + /// + /// # Panics + /// + /// Panics if `size` is `0`. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// + /// let config = Config::default().batch_size(50); + /// + /// assert_eq!(config.batch_size, 50); + /// ``` + #[must_use] + pub fn batch_size(mut self, size: usize) -> Self { + assert!(size > 0, "batch size cannot be 0"); + self.batch_size = size; + self + } + + /// Sets the interval between worker heartbeats. + /// + /// A shorter interval detects failed workers sooner but produces + /// heartbeat activity more frequently. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// use std::time::Duration; + /// + /// let config = Config::default() + /// .heartbeat_interval(Duration::from_secs(15)); + /// + /// assert_eq!(config.heartbeat_interval, Duration::from_secs(15)); + /// ``` + #[must_use] + pub fn heartbeat_interval(mut self, interval: Duration) -> Self { + self.heartbeat_interval = interval; + self + } + + /// Sets the queue from which jobs are consumed. + /// + /// # Examples + /// + /// ``` + /// # use apalis_postgres::Config; + /// let config = Config::default() + /// .queue("high-priority"); + /// + /// assert_eq!(config.queue.as_ref(), "high-priority"); + /// ``` + #[must_use] + pub fn queue(mut self, queue: impl AsRef) -> Self { + self.queue = Queue::from(queue.as_ref()); + self + } + + /// Sets the number of missed heartbeats allowed before a worker is + /// considered dead. + /// + /// This value works together with [`Self::heartbeat_interval`]. + /// For example, a 30-second heartbeat interval with `2` missed + /// heartbeats results in an orphan timeout of 60 seconds. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// use std::time::Duration; + /// + /// let config = Config::default().missed_heartbeats(3); + /// + /// assert_eq!(config.missed_heartbeats, 3); + /// assert_eq!( + /// config.orphaned_duration(), + /// Duration::from_secs(90) + /// ); + /// ``` + #[must_use] + pub fn missed_heartbeats(mut self, missed_heartbeats: usize) -> Self { + self.missed_heartbeats = missed_heartbeats; + self + } + + /// Sets the database URL used by the worker. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// + /// let config = Config::default() + /// .database_url("postgres://localhost/apalis"); + /// + /// assert_eq!( + /// config.database_url.as_deref(), + /// Some("postgres://localhost/apalis") + /// ); + /// ``` + #[must_use] + pub fn database_url(mut self, database_url: impl Into) -> Self { + self.database_url = Some(database_url.into()); + self + } + + /// Enables or disables task locking. + /// + /// When enabled, tasks are locked while being processed to prevent + /// multiple workers from processing the same task concurrently. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// + /// let config = Config::default().lock_tasks(false); + /// + /// assert!(!config.lock_tasks); + /// ``` + #[must_use] + pub fn lock_tasks(mut self, lock_tasks: bool) -> Self { + self.lock_tasks = lock_tasks; + self + } + + /// Enables or disables result persistence. + /// + /// When enabled, results produced by completed jobs are persisted. + /// + /// # Examples + /// + /// ``` + /// use apalis_postgres::Config; + /// + /// let config = Config::default().persist_results(false); + /// + /// assert!(!config.persist_results); + /// ``` + #[must_use] + pub fn persist_results(mut self, persist_results: bool) -> Self { + self.persist_results = persist_results; + self + } + + /// Returns the amount of time after which a worker may be considered + /// orphaned. + /// + /// The duration is calculated as: + /// + /// ```text + /// heartbeat_interval × missed_heartbeats + /// ``` + #[must_use] + pub fn orphaned_duration(&self) -> Duration { + self.heartbeat_interval * self.missed_heartbeats as u32 + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..a73121b --- /dev/null +++ b/src/error.rs @@ -0,0 +1,33 @@ +use apalis_core::task::{status::StatusError, task_id::TaskIdError}; +use sqlx::Error as SqlxError; +/// Represents a wrapper for errors encountered on this crate +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Inner engine error + #[error(transparent)] + Database(#[from] SqlxError), + /// Error handling json + #[error("JsonError: {0}")] + JsonError(serde_json::Error), + /// Reenqueue Mismatch error + #[error("ReenqueueMismatch: Queued [{queued}] , Abandoned[{abandoned}] ")] + ReenqueueMismatch { + /// The db count + queued: usize, + /// The workers count + abandoned: usize, + }, + /// Error decoding the task_id + #[error("TaskIdError: {0}")] + TaskIdError(TaskIdError), + /// Error decoding the task status + #[error("StatusError: {0}")] + StatusError(StatusError), + /// Worker was removed in the database + #[error("WorkerOutOfSync")] + WorkerOutOfSync, + + /// Tried to register a worker that already exists + #[error("WorkerAlreadyExists: {0}")] + WorkerAlreadyExists(String), +} diff --git a/src/factory.rs b/src/factory.rs new file mode 100644 index 0000000..475a467 --- /dev/null +++ b/src/factory.rs @@ -0,0 +1,297 @@ +//! PostgreSQL backend factory with shared `LISTEN/NOTIFY` polling. +//! +//! [`PostgresStorageFactory`] creates independent PostgreSQL backends while +//! sharing a single PostgreSQL notification listener. Each backend is +//! registered by its queue name and receives task IDs for newly inserted +//! jobs belonging to that queue. +//! +//! This avoids creating a separate [`PgListener`] for every worker and allows +//! multiple queue types to share the same database connection pool. +//! +//! The factory creates [`PostgresStorage`] instances configured with +//! [`StreamStrategy`]. PostgreSQL notifications wake the corresponding +//! backend, which then performs its normal database polling. +//! +//! # Example +//! +//! ```no_run +//! use apalis::prelude::*; +//! use apalis_postgres::factory::PostgresStorageFactory; +//! use sqlx::PgPool; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box> { +//! let pool = PgPool::connect( +//! &std::env::var("DATABASE_URL")? +//! ).await?; +//! +//! let mut factory = PostgresStorageFactory::new(pool); +//! +//! let mut backend = factory.create()?; +//! +//! backend.push(42).await?; +//! +//! let worker = WorkerBuilder::new("numbers") +//! .backend(backend) +//! .build(|task: u64| async move { +//! println!("processing {task}"); +//! }); +//! +//! worker.run().await?; +//! Ok(()) +//! } +//! ``` +//! +//! A factory can also create multiple queues. Each queue is independently +//! notified when a matching job is inserted: +//! +//! ```ignore +//! # use apalis::prelude::*; +//! # use apalis_postgres::factory::PostgresStorageFactory; +//! # use sqlx::PgPool; +//! # async fn example(pool: PgPool) -> Result<(), Box> { +//! let mut factory = PostgresStorageFactory::new(pool); +//! +//! let emails = factory.create::()?; +//! let reports = factory.create::()?; +//! # Ok(()) +//! # } +//! # struct Email; +//! # struct Report; +//! ``` +//! +//! [`PostgresStorage`]: crate::PostgresStorage +//! [`PgListener`]: sqlx::postgres::PgListener +//! [`StreamStrategy`]: apalis_core::backend::ext::poll_strategy::StreamStrategy +use std::{ + collections::HashMap, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use crate::{PgTaskId, PostgresStorage, config::Config, pubsub::InsertEvent}; +use apalis_core::backend::{ + BackendConfig, + ext::{ + BackendExt, + poll_strategy::{PollWith, StreamStrategy}, + }, + factory::BackendFactory, +}; + +use futures::{ + FutureExt, SinkExt, Stream, StreamExt, + channel::mpsc::{self, Receiver, Sender}, + future::{BoxFuture, Shared}, + lock::Mutex, +}; +use sqlx::{PgPool, postgres::PgListener}; + +/// A factory for creating PostgreSQL-backed task queues. +/// +/// `PostgresStorageFactory` maintains a shared PostgreSQL `LISTEN` connection +/// and routes job insertion notifications to the corresponding queue. +/// +/// Each queue created by the factory is identified by its queue name. Multiple +/// backends can therefore share a single notification listener while retaining +/// independent task polling and processing. +/// +/// # Example +/// +/// ```ignore +/// use apalis_core::backend::factory::BackendFactory; +/// use apalis_postgres::factory::PostgresStorageFactory; +/// use sqlx::PgPool; +/// +/// # async fn example() -> Result<(), Box> { +/// let pool = PgPool::connect("postgres://localhost/apalis").await?; +/// let mut factory = PostgresStorageFactory::new(pool); +/// +/// let backend = factory.create::()?; +/// # let _ = backend; +/// # Ok(()) +/// # } +/// ``` +pub struct PostgresStorageFactory { + pool: PgPool, + registry: Arc>>>, + drive: Shared>, +} + +impl PostgresStorageFactory { + /// Creates a new factory backed by the given PostgreSQL connection pool. + /// + /// A single PostgreSQL notification listener is shared by all backends + /// created by this factory. + pub fn new(pool: PgPool) -> Self { + let registry: Arc>>> = + Arc::new(Mutex::new(HashMap::default())); + let p = pool.clone(); + let instances = registry.clone(); + Self { + pool, + drive: async move { + let mut listener = PgListener::connect_with(&p).await.unwrap(); + listener.listen("apalis::job::insert").await.unwrap(); + listener + .into_stream() + .filter_map(|notification| { + let instances = instances.clone(); + async move { + let pg_notification = notification.ok()?; + let payload = pg_notification.payload(); + let ev: InsertEvent = serde_json::from_str(payload).ok()?; + let instances = instances.lock().await; + if instances.get(&ev.job_type).is_some() { + return Some(ev); + } + None + } + }) + .for_each(|ev| { + let instances = instances.clone(); + async move { + let mut instances = instances.lock().await; + let sender = instances.get_mut(&ev.job_type).unwrap(); + sender.send(ev.id).await.unwrap(); + } + }) + .await; + } + .boxed() + .shared(), + registry, + } + } +} + +/// Errors returned when creating a PostgreSQL backend from a +/// [`PostgresStorageFactory`]. +/// +/// [`PostgresStorageFactory`]: crate::factory::PostgresStorageFactory +#[derive(Debug, thiserror::Error)] +pub enum PostgresFactoryError { + /// Namespace not found + #[error("namespace already exists: {0}")] + NamespaceExists(String), + + /// Registry locked + #[error("registry locked")] + RegistryLocked, +} + +impl BackendFactory for PostgresStorageFactory { + type Backend = PollWith, StreamStrategy>; + type Error = PostgresFactoryError; + + fn create(&mut self) -> Result + where + ::Config: Default, + { + self.create_with_config(Config::default().queue(std::any::type_name::())) + } + fn create_with_config(&mut self, config: Config) -> Result { + let mut registry = self + .registry + .try_lock() + .ok_or(PostgresFactoryError::RegistryLocked)?; + + let (tx, rx) = mpsc::channel(config.batch_size * registry.len()); + if registry.insert(config.queue.to_string(), tx).is_some() { + return Err(PostgresFactoryError::NamespaceExists( + config.queue.to_string(), + )); + } + Ok(PostgresStorage::new(&self.pool) + .with_config(config) + .poll_with_stream(SharedFetcher { + poller: self.drive.clone(), + receiver: Arc::new(Mutex::new(rx)), + })) + } +} + +/// A stream of task IDs received from the shared PostgreSQL notification +/// listener. +/// +/// `SharedFetcher` keeps the shared notification driver alive while exposing +/// notifications for a specific queue as a [`Stream`]. +/// +/// The fetcher does not perform database polling itself. Instead, it yields +/// task IDs received through PostgreSQL `LISTEN/NOTIFY`, allowing the backend's +/// polling strategy to use those notifications as a wake-up signal. +/// +/// [`Stream`]: futures::Stream +#[derive(Clone, Debug)] +pub struct SharedFetcher { + poller: Shared>, + receiver: Arc>>, +} + +impl Stream for SharedFetcher { + type Item = PgTaskId; + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + // Keep the poller alive by polling it, but ignoring the output + let _ = this.poller.poll_unpin(cx); + + // Delegate actual items to receiver + let mut receiver = this.receiver.try_lock(); + if let Some(ref mut rx) = receiver { + rx.poll_next_unpin(cx) + } else { + Poll::Pending + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use apalis_core::{ + backend::TaskSink, + error::BoxDynError, + worker::{builder::WorkerBuilder, context::WorkerContext}, + }; + use futures::stream; + + use super::*; + + #[tokio::test] + async fn basic_worker() { + let pool = PgPool::connect(std::env::var("DATABASE_URL").unwrap().as_str()) + .await + .unwrap(); + let mut store = PostgresStorageFactory::new(pool); + + let mut map_store = store.create().unwrap(); + + let mut int_store = store.create().unwrap(); + + map_store + .push_stream(&mut stream::iter(vec![HashMap::::new()])) + .await + .unwrap(); + int_store.push(99).await.unwrap(); + + async fn send_reminder( + _: T, + _task_id: PgTaskId, + wrk: WorkerContext, + ) -> Result<(), BoxDynError> { + tokio::time::sleep(Duration::from_secs(2)).await; + wrk.stop().unwrap(); + Ok(()) + } + + let int_worker = WorkerBuilder::new("rango-tango-3") + .backend(int_store) + .build(send_reminder); + let map_worker = WorkerBuilder::new("rango-tango-4") + .backend(map_store) + .build(send_reminder); + tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); + } +} diff --git a/src/fetcher.rs b/src/fetcher.rs deleted file mode 100644 index c809da1..0000000 --- a/src/fetcher.rs +++ /dev/null @@ -1,172 +0,0 @@ -use std::{ - collections::VecDeque, - marker::PhantomData, - pin::Pin, - task::{Context, Poll}, - time::{Duration, Instant}, -}; - -use apalis_core::{task::Task, timer::Delay, worker::context::WorkerContext}; -use apalis_sql::from_row::TaskRow; -use futures::{Future, FutureExt, future::BoxFuture, stream::Stream}; -use pin_project::pin_project; - -use sqlx::{PgPool, Pool, Postgres}; -use ulid::Ulid; - -use crate::{CompactType, Config, PgContext, PgTask, from_row::PgTaskRow}; - -async fn fetch_next( - pool: PgPool, - config: Config, - worker: WorkerContext, -) -> Result>, sqlx::Error> { - let job_type = config.queue().to_string(); - let buffer_size = config.buffer_size() as i32; - - sqlx::query_file_as!( - PgTaskRow, - "queries/task/fetch_next.sql", - worker.name(), - job_type, - buffer_size - ) - .fetch_all(&pool) - .await? - .into_iter() - .map(|r| { - let row: TaskRow = r.try_into()?; - row.try_into_task_compact() - .map_err(|e| sqlx::Error::Protocol(e.to_string())) - }) - .collect() -} - -enum StreamState { - Ready, - Delay(Delay), - Fetch(BoxFuture<'static, Result>, sqlx::Error>>), - Buffered(VecDeque>), -} - -/// Dispatcher for fetching tasks from a PostgreSQL backend via [PgPollFetcher] -#[derive(Clone, Debug)] -pub struct PgFetcher { - pub _marker: PhantomData<(Compact, Decode)>, -} - -#[pin_project] -pub struct PgPollFetcher { - pool: PgPool, - config: Config, - wrk: WorkerContext, - #[pin] - state: StreamState, - current_backoff: Duration, - last_fetch_time: Option, -} - -impl Clone for PgPollFetcher { - fn clone(&self) -> Self { - Self { - pool: self.pool.clone(), - config: self.config.clone(), - wrk: self.wrk.clone(), - state: StreamState::Ready, - current_backoff: self.current_backoff, - last_fetch_time: self.last_fetch_time, - } - } -} - -impl PgPollFetcher { - pub fn new(pool: &Pool, config: &Config, wrk: &WorkerContext) -> Self { - let initial_backoff = Duration::from_secs(1); - Self { - pool: pool.clone(), - config: config.clone(), - wrk: wrk.clone(), - state: StreamState::Ready, - current_backoff: initial_backoff, - last_fetch_time: None, - } - } -} - -impl Stream for PgPollFetcher { - type Item = Result>, sqlx::Error>; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - loop { - match this.state { - StreamState::Ready => { - let stream = - fetch_next(this.pool.clone(), this.config.clone(), this.wrk.clone()); - this.state = StreamState::Fetch(stream.boxed()); - } - StreamState::Delay(ref mut delay) => match Pin::new(delay).poll(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(_) => this.state = StreamState::Ready, - }, - - StreamState::Fetch(ref mut fut) => match fut.poll_unpin(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(item) => match item { - Ok(requests) => { - if requests.is_empty() { - let next = this.next_backoff(this.current_backoff); - this.current_backoff = next; - let delay = Delay::new(this.current_backoff); - this.state = StreamState::Delay(delay); - } else { - let mut buffer = VecDeque::new(); - for request in requests { - buffer.push_back(request); - } - this.current_backoff = Duration::from_secs(1); - this.state = StreamState::Buffered(buffer); - } - } - Err(e) => { - let next = this.next_backoff(this.current_backoff); - this.current_backoff = next; - this.state = StreamState::Delay(Delay::new(next)); - return Poll::Ready(Some(Err(e))); - } - }, - }, - - StreamState::Buffered(ref mut buffer) => { - if let Some(request) = buffer.pop_front() { - // Yield the next buffered item - if buffer.is_empty() { - // Buffer is now empty, transition to ready for next fetch - this.state = StreamState::Ready; - } - return Poll::Ready(Some(Ok(Some(request)))); - } else { - // Buffer is empty, transition to ready - this.state = StreamState::Ready; - } - } - } - } - } -} - -impl PgPollFetcher { - fn next_backoff(&self, current: Duration) -> Duration { - let doubled = current * 2; - std::cmp::min(doubled, Duration::from_secs(60 * 5)) - } - - #[allow(unused)] - pub fn take_pending(&mut self) -> VecDeque> { - match &mut self.state { - StreamState::Buffered(tasks) => std::mem::take(tasks), - _ => VecDeque::new(), - } - } -} diff --git a/src/from_row.rs b/src/from_row.rs index b197386..74d80f5 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -1,4 +1,15 @@ -use apalis_sql::{DateTime, TaskRow}; +use std::{collections::HashMap, str::FromStr}; + +use apalis_core::task::{ + builder::TaskBuilder, + metadata::MetadataStore, + status::Status, + task_id::{TaskId, TaskIdError::Decode}, +}; +use sqlx::postgres::types::PgHstore; +use ulid::Ulid; + +use crate::{PgTask, error::Error, timestamp::Timestamp}; #[derive(Debug)] pub struct PgTaskRow { @@ -8,43 +19,77 @@ pub struct PgTaskRow { pub status: Option, pub attempts: Option, pub max_attempts: Option, - pub run_at: Option, + pub run_at: Option, + #[allow(unused)] pub last_result: Option, - pub lock_at: Option, + pub lock_at: Option, pub lock_by: Option, - pub done_at: Option, + pub done_at: Option, pub priority: Option, pub idempotency_key: Option, - pub metadata: Option, + pub metadata: Option, } -impl TryInto for PgTaskRow { - type Error = sqlx::Error; - fn try_into(self) -> Result { - Ok(TaskRow { - job: self.job.unwrap_or_default(), - id: self +impl TryInto>> for PgTaskRow { + type Error = Error; + + fn try_into(self) -> Result>, Self::Error> { + let mut task = TaskBuilder::new( + self.job + .ok_or_else(|| sqlx::Error::ColumnNotFound("job".into()))?, + ) + .task_id({ + let task_id = self .id - .ok_or_else(|| sqlx::Error::Protocol("Missing id".into()))?, - job_type: self - .job_type - .ok_or_else(|| sqlx::Error::Protocol("Missing job_type".into()))?, - status: self - .status - .ok_or_else(|| sqlx::Error::Protocol("Missing status".into()))?, - attempts: self - .attempts - .ok_or_else(|| sqlx::Error::Protocol("Missing attempts".into()))? - as usize, - max_attempts: self.max_attempts.map(|v| v as usize), - run_at: self.run_at, - last_result: self.last_result, - lock_at: self.lock_at, - lock_by: self.lock_by, - done_at: self.done_at, - priority: self.priority.map(|v| v as usize), - idempotency_key: self.idempotency_key, - metadata: self.metadata, + .ok_or_else(|| sqlx::Error::ColumnNotFound("task_id".into()))?; + TaskId::from_ulid( + Ulid::from_string(&task_id) + .map_err(|e| Error::TaskIdError(Decode(e.to_string())))?, + ) }) + .queue( + self.job_type + .ok_or_else(|| sqlx::Error::ColumnNotFound("job_type".into()))? + .into(), + ) + .status( + Status::from_str( + &self + .status + .ok_or_else(|| sqlx::Error::ColumnNotFound("status".into()))?, + ) + .map_err(Error::StatusError)?, + ) + .attempt( + self.attempts + .ok_or_else(|| sqlx::Error::ColumnNotFound("attempts".into()))? + as usize, + ) + .max_attempts(self.max_attempts.map(|v| v as usize).unwrap_or(25)) + .run_at_timestamp( + self.run_at + .ok_or(sqlx::Error::ColumnNotFound("run_at".to_owned()))? + .0, + ) + .lock_at(self.lock_at.map(|dt| dt.0)) + .done_at(self.done_at.map(|dt| dt.0)) + .lock_by(self.lock_by) + .priority(self.priority.map(|v| v as usize).unwrap_or_default()) + .with_metadata( + self.metadata + .map(|meta| { + meta.into_iter() + .map(|(k, v)| (k, v.unwrap())) + .collect::>() + }) + .map(MetadataStore::from_map) + .unwrap_or_default(), + ); + + if let Some(idempotency_key) = self.idempotency_key { + task = task.idempotency_key(idempotency_key); + } + + Ok(task.build()) } } diff --git a/src/lib.rs b/src/lib.rs index 5f6a722..244ce37 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,479 +1,42 @@ #![doc = include_str!("../README.md")] //! //! [`PostgresStorageWithListener`]: crate::PostgresStorage -//! [`SharedPostgresStorage`]: crate::shared::SharedPostgresStorage -use std::{fmt::Debug, marker::PhantomData}; +//! [`PostgresStorageFactory`]: crate::factory::PostgresStorageFactory -pub use apalis_codec::json::JsonCodec; -use apalis_core::{ - backend::{Backend, BackendExt, TaskStream, codec::Codec, queue::Queue}, - features_table, - layers::Stack, - task::{Task, task_id::TaskId}, - worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, -}; -pub use apalis_sql::{config::Config, from_row::TaskRow}; -use futures::{ - StreamExt, TryFutureExt, TryStreamExt, - future::ready, - stream::{self, BoxStream, select}, -}; -use serde::Deserialize; -pub use sqlx::{PgPool, postgres::PgConnectOptions, postgres::PgListener, postgres::Postgres}; -use ulid::Ulid; - -pub use crate::{ - ack::{LockTaskLayer, PgAck}, - fetcher::{PgFetcher, PgPollFetcher}, - queries::{ - keep_alive::{initial_heartbeat, keep_alive_stream}, - reenqueue_orphaned::reenqueue_orphaned_stream, - }, - sink::PgSink, -}; - -mod ack; -mod fetcher; +use apalis_core::task::{Task, task_id::TaskId}; +mod backend; +mod config; +mod error; +pub mod factory; mod from_row; - -pub type PgContext = apalis_sql::context::SqlContext; -mod queries; -pub mod shared; -pub mod sink; - -pub type PgTask = Task; - -pub type PgTaskId = TaskId; - -pub type CompactType = Vec; - -#[doc = features_table! { - setup = r#" - # { - # use apalis_postgres::PostgresStorage; - # use sqlx::PgPool; - # let pool = PgPool::connect(std::env::var("DATABASE_URL").unwrap().as_str()).await.unwrap(); - # PostgresStorage::setup(&pool).await.unwrap(); - # PostgresStorage::new(&pool) - # }; - "#, - - Backend => supported("Supports storage and retrieval of tasks", true), - TaskSink => supported("Ability to push new tasks", true), - Serialization => supported("Serialization support for arguments", true), - Workflow => supported("Flexible enough to support workflows", true), - WebUI => supported("Expose a web interface for monitoring tasks", true), - FetchById => supported("Allow fetching a task by its ID", false), - RegisterWorker => supported("Allow registering a worker with the backend", false), - MakeShared => supported("Share one connection across multiple workers via [`SharedPostgresStorage`]", false), - WaitForCompletion => supported("Wait for tasks to complete without blocking", true), - ResumeById => supported("Resume a task by its ID", false), - ResumeAbandoned => supported("Resume abandoned tasks", false), - ListWorkers => supported("List all workers registered with the backend", false), - ListTasks => supported("List all tasks in the backend", false), -}] -/// -/// [`SharedPostgresStorage`]: crate::shared::SharedPostgresStorage -#[pin_project::pin_project] -pub struct PostgresStorage< - Args, - Compact = CompactType, - Codec = JsonCodec, - Fetcher = PgFetcher, -> { - _marker: PhantomData<(Args, Compact, Codec)>, - pool: PgPool, - config: Config, - #[pin] - fetcher: Fetcher, - #[pin] - sink: PgSink, -} - -/// A fetcher that does nothing, used for notify-based storage -#[derive(Debug, Clone, Default)] -pub struct PgNotify { - _private: PhantomData<()>, -} - -impl Clone - for PostgresStorage -{ - fn clone(&self) -> Self { - Self { - _marker: PhantomData, - pool: self.pool.clone(), - config: self.config.clone(), - fetcher: self.fetcher.clone(), - sink: self.sink.clone(), - } - } -} - -impl PostgresStorage<(), (), ()> { - /// Perform migrations for storage. - /// - /// On an existing (pre-`1.0`) database, run the one-time transition documented - /// under "Upgrading to 1.0" in the README before calling this — `setup()` no - /// longer relocates the migration history automatically. Fresh databases need - /// no manual steps. - #[cfg(feature = "migrate")] - pub async fn setup(pool: &PgPool) -> Result<(), sqlx::Error> { - Self::migrations().run(pool).await?; - Ok(()) - } - - /// Get postgres migrations without running them - #[cfg(feature = "migrate")] - pub fn migrations() -> sqlx::migrate::Migrator { - sqlx::migrate!("./migrations") - } -} - -impl PostgresStorage { - pub fn new(pool: &PgPool) -> Self { - let config = Config::new(std::any::type_name::()); - Self::new_with_config(pool, &config) - } - - /// Creates a new PostgresStorage instance. - pub fn new_with_config(pool: &PgPool, config: &Config) -> Self { - let sink = PgSink::new(pool, config); - Self { - _marker: PhantomData, - pool: pool.clone(), - config: config.clone(), - fetcher: PgFetcher { - _marker: PhantomData, - }, - sink, - } - } - - pub fn new_with_notify( - pool: &PgPool, - config: &Config, - ) -> PostgresStorage, PgNotify> { - let sink = PgSink::new(pool, config); - - PostgresStorage { - _marker: PhantomData, - pool: pool.clone(), - config: config.clone(), - fetcher: PgNotify::default(), - sink, - } - } - - /// Returns a reference to the pool. - pub fn pool(&self) -> &PgPool { - &self.pool - } - - /// Returns a reference to the config. - pub fn config(&self) -> &Config { - &self.config - } -} - -impl PostgresStorage { - pub fn with_codec(self) -> PostgresStorage { - PostgresStorage { - _marker: PhantomData, - sink: PgSink::new(&self.pool, &self.config), - pool: self.pool, - config: self.config, - fetcher: self.fetcher, - } - } -} - -impl Backend - for PostgresStorage> -where - Args: Send + 'static + Unpin, - Decode: Codec + Send + 'static, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Args = Args; - - type IdType = Ulid; - - type Context = PgContext; - - type Error = sqlx::Error; - - type Stream = TaskStream, sqlx::Error>; - - type Beat = BoxStream<'static, Result<(), sqlx::Error>>; - - type Layer = Stack>; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let pool = self.pool.clone(); - let config = self.config.clone(); - let worker = worker.clone(); - let keep_alive = keep_alive_stream(pool, config, worker); - let reenqueue = reenqueue_orphaned_stream( - self.pool.clone(), - self.config.clone(), - *self.config.keep_alive(), - ) - .map_ok(|_| ()); - futures::stream::select(keep_alive, reenqueue).boxed() - } - - fn middleware(&self) -> Self::Layer { - Stack::new( - LockTaskLayer::new(self.pool.clone()), - AcknowledgeLayer::new(PgAck::new(self.pool.clone())), - ) - } - - fn poll(self, worker: &WorkerContext) -> Self::Stream { - self.poll_basic(worker) - .map(|a| match a { - Ok(Some(task)) => Ok(Some( - task.try_map(|t| Decode::decode(&t)) - .map_err(|e| sqlx::Error::Decode(e.into()))?, - )), - Ok(None) => Ok(None), - Err(e) => Err(e), - }) - .boxed() - } -} - -impl BackendExt - for PostgresStorage> -where - Args: Send + 'static + Unpin, - Decode: Codec + Send + 'static, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Compact = CompactType; - - type Codec = Decode; - type CompactStream = TaskStream, Self::Error>; - - fn get_queue(&self) -> Queue { - self.config.queue().clone() - } - - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream { - self.poll_basic(worker).boxed() - } -} - -impl PostgresStorage> -where - Args: Send + 'static + Unpin, -{ - fn poll_basic(&self, worker: &WorkerContext) -> TaskStream, sqlx::Error> { - let register_worker = initial_heartbeat( - self.pool.clone(), - self.config.clone(), - worker.clone(), - "PostgresStorage", - ) - .map_ok(|_| None); - let register = stream::once(register_worker); - register - .chain(PgPollFetcher::::new( - &self.pool, - &self.config, - worker, - )) - .boxed() - } -} - -impl Backend for PostgresStorage -where - Args: Send + 'static + Unpin, - Decode: Codec + 'static + Send, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Args = Args; - - type IdType = Ulid; - - type Context = PgContext; - - type Error = sqlx::Error; - - type Stream = TaskStream, sqlx::Error>; - - type Beat = BoxStream<'static, Result<(), sqlx::Error>>; - - type Layer = Stack>; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let pool = self.pool.clone(); - let config = self.config.clone(); - let worker = worker.clone(); - let keep_alive = keep_alive_stream(pool, config, worker); - let reenqueue = reenqueue_orphaned_stream( - self.pool.clone(), - self.config.clone(), - *self.config.keep_alive(), - ) - .map_ok(|_| ()); - futures::stream::select(keep_alive, reenqueue).boxed() - } - - fn middleware(&self) -> Self::Layer { - Stack::new( - LockTaskLayer::new(self.pool.clone()), - AcknowledgeLayer::new(PgAck::new(self.pool.clone())), - ) - } - - fn poll(self, worker: &WorkerContext) -> Self::Stream { - self.poll_with_notify(worker) - .map(|a| match a { - Ok(Some(task)) => Ok(Some( - task.try_map(|t| Decode::decode(&t)) - .map_err(|e| sqlx::Error::Decode(e.into()))?, - )), - Ok(None) => Ok(None), - Err(e) => Err(e), - }) - .boxed() - } -} - -impl BackendExt for PostgresStorage -where - Args: Send + 'static + Unpin, - Decode: Codec + 'static + Unpin + Send, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Compact = CompactType; - - type Codec = Decode; - type CompactStream = TaskStream, Self::Error>; - - fn get_queue(&self) -> Queue { - self.config.queue().clone() - } - - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream { - self.poll_with_notify(worker).boxed() - } -} - -impl PostgresStorage { - pub fn poll_with_notify( - &self, - worker: &WorkerContext, - ) -> TaskStream, sqlx::Error> { - let pool = self.pool.clone(); - let worker_id = worker.name().to_owned(); - let namespace = self.config.queue().to_string(); - let listener = async move { - let mut fetcher = PgListener::connect_with(&pool) - .await - .expect("Failed to create listener"); - fetcher.listen("apalis::job::insert").await.unwrap(); - fetcher - }; - let fetcher = stream::once(listener).flat_map(|f| f.into_stream()); - let pool = self.pool.clone(); - let register_worker = initial_heartbeat( - self.pool.clone(), - self.config.clone(), - worker.clone(), - "PostgresStorageWithNotify", - ) - .map_ok(|_| None); - let register = stream::once(register_worker); - let lazy_fetcher = fetcher - .into_stream() - .filter_map(move |notification| { - let namespace = namespace.clone(); - async move { - let pg_notification = notification.ok()?; - let payload = pg_notification.payload(); - let ev: InsertEvent = serde_json::from_str(payload).ok()?; - - if ev.job_type == namespace { - return Some(ev.id); - } - None - } - }) - .map(|t| t.to_string()) - .ready_chunks(self.config.buffer_size()) - .then(move |ids| { - let pool = pool.clone(); - let worker_id = worker_id.clone(); - async move { - let mut tx = pool.begin().await?; - use crate::from_row::PgTaskRow; - let res: Vec<_> = sqlx::query_file_as!( - PgTaskRow, - "queries/task/queue_by_id.sql", - &ids, - &worker_id - ) - .fetch(&mut *tx) - .map(|r| { - let row: TaskRow = r?.try_into()?; - Ok(Some( - row.try_into_task_compact() - .map_err(|e| sqlx::Error::Protocol(e.to_string()))?, - )) - }) - .collect() - .await; - tx.commit().await?; - Ok::<_, sqlx::Error>(res) - } - }) - .flat_map(|vec| match vec { - Ok(vec) => stream::iter(vec.into_iter().map(|res| match res { - Ok(t) => Ok(t), - Err(e) => Err(e), - })) - .boxed(), - Err(e) => stream::once(ready(Err(e))).boxed(), - }) - .boxed(); - - let eager_fetcher = StreamExt::boxed(PgPollFetcher::::new( - &self.pool, - &self.config, - worker, - )); - register.chain(select(lazy_fetcher, eager_fetcher)).boxed() - } -} - -#[derive(Debug, Deserialize)] -pub struct InsertEvent { - job_type: String, - id: PgTaskId, -} +mod persistence; +mod pubsub; +pub mod queries; +mod sink; +mod timestamp; + +pub use config::Config; +pub use error::Error; +pub use pubsub::{InsertEvent, Pubsub}; + +/// An alias for [Task], specialized for Postgres. +pub type PgTask> = Task; +/// An alias for [TaskId] using [TaskId::Ulid], specialized for Postgres. +pub type PgTaskId = TaskId; +pub use crate::backend::PostgresStorage; +pub use sqlx::{PgPool, postgres::PgConnectOptions, postgres::PgConnection, postgres::PgListener}; #[cfg(test)] mod tests { - use std::{ - collections::HashMap, - env, - time::{Duration, Instant}, - }; - - use apalis_workflow::Workflow; - use apalis_workflow::WorkflowSink; + use std::{collections::HashMap, env, time::Duration}; - use apalis_core::{ - backend::poll_strategy::{IntervalStrategy, StrategyBuilder}, - error::BoxDynError, - task::data::Data, - worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, - }; + use apalis_workflow::SteppedFlow; + use futures::{StreamExt, stream}; use serde::{Deserialize, Serialize}; + use sqlx::PgPool; + + use crate::config::Config; + use apalis::prelude::*; use super::*; @@ -483,7 +46,11 @@ mod tests { let pool = PgPool::connect(env::var("DATABASE_URL").unwrap().as_str()) .await .unwrap(); - let mut backend = PostgresStorage::new(&pool); + let config = Config::default() + .queue("sample") + .batch_size(50) + .lock_tasks(false); + let mut backend = PostgresStorage::new(&pool).with_config(config); let mut items = stream::repeat_with(HashMap::default).take(1); backend.push_stream(&mut items).await.unwrap(); @@ -502,31 +69,26 @@ mod tests { .build(send_reminder); worker.run().await.unwrap(); } - #[tokio::test] async fn notify_worker() { - use apalis_core::backend::TaskSink; let pool = PgPool::connect(env::var("DATABASE_URL").unwrap().as_str()) .await .unwrap(); - let config = Config::new("test").with_poll_interval( - StrategyBuilder::new() - .apply(IntervalStrategy::new(Duration::from_secs(6))) - .build(), - ); - let backend = PostgresStorage::new_with_notify(&pool, &config); + let config = Config::default() + .queue("test") + .persist_results(true) + .lock_tasks(false) + .batch_size(20); + let backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub(); let mut b = backend.clone(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(2)).await; - let mut items = stream::repeat_with(|| { - Task::builder(42u32) - .with_ctx(PgContext::new().with_priority(1)) - .build() - }) - .take(1); - b.push_all(&mut items).await.unwrap(); + tokio::time::sleep(Duration::from_secs(3)).await; + let task = TaskBuilder::new(42u32).priority(1).build(); + b.push_task(task).await.unwrap(); }); async fn send_reminder(_: u32, wrk: WorkerContext) -> Result<(), BoxDynError> { @@ -534,12 +96,12 @@ mod tests { Ok(()) } - let instant = Instant::now(); - let worker = WorkerBuilder::new("rango-tango-2") + let ctx = WorkerContext::new("rango-tango-2"); + let worker = WorkerBuilder::new(&ctx) .backend(backend) .build(send_reminder); worker.run().await.unwrap(); - let run_for = instant.elapsed(); + let run_for = ctx.elapsed(); assert!( run_for < Duration::from_secs(4), "Worker did not use notify mechanism" @@ -572,13 +134,10 @@ mod tests { sentiment: Option, } - let workflow = Workflow::new("text-pipeline") + let workflow = SteppedFlow::new("text-pipeline") // Step 1: Preprocess input (e.g., tokenize, lowercase) - .and_then(|input: UserInput, mut worker: WorkerContext| async move { - worker.emit(&Event::Custom(Box::new(format!( - "Preprocessing input: {}", - input.text - )))); + .and_then(|input: UserInput, worker: WorkerContext| async move { + worker.emit(format!("Preprocessing input: {}", input.text)); let processed = input.text.to_lowercase(); Ok::<_, BoxDynError>(processed) }) @@ -627,29 +186,24 @@ mod tests { }) } }) - .and_then(|a: Vec, mut worker: WorkerContext| async move { + .and_then(|a: Vec, worker: WorkerContext| async move { dbg!(&a); - worker.emit(&Event::Custom(Box::new(format!( - "Generated {} summaries", - a.len() - )))); + worker.emit(format!("Generated {} summaries", a.len())); worker.stop() }); let pool = PgPool::connect(env::var("DATABASE_URL").unwrap().as_str()) .await .unwrap(); - let config = Config::new("test").with_poll_interval( - StrategyBuilder::new() - .apply(IntervalStrategy::new(Duration::from_secs(1))) - .build(), - ); - let mut backend = PostgresStorage::new_with_notify(&pool, &config); + let config = Config::default().queue("test"); + let mut backend = PostgresStorage::new(&pool) + .with_config(config) + .with_pubsub(); let input = UserInput { text: "Rust makes systems programming delightful!".to_string(), }; - backend.push_start(input).await.unwrap(); + backend.push(input).await.unwrap(); let worker = WorkerBuilder::new("rango-tango") .backend(backend) diff --git a/src/persistence.rs b/src/persistence.rs new file mode 100644 index 0000000..d983fbe --- /dev/null +++ b/src/persistence.rs @@ -0,0 +1,166 @@ +use std::{ + collections::HashSet, + time::{SystemTime, UNIX_EPOCH}, +}; + +use apalis_core::{ + backend::persistence::{Persistence, TaskEvent}, + worker::context::WorkerContext, +}; +use serde_json::Value; +use sqlx::PgPool; + +use crate::{ + PgTask, + config::Config, + error::Error, + queries::{ + self, fetch_next, keep_alive, reenqueue_abandoned, reenqueue_orphaned, register_worker, + }, + sink::push_tasks, + timestamp::Timestamp, +}; + +#[derive(Debug, Clone)] +pub(crate) struct SqlxPersistence { + pub(crate) pool: PgPool, + pub(crate) config: Config, +} + +impl Persistence for SqlxPersistence { + type Compact = Vec; + type Error = Error; + type Response = Value; + async fn register(&mut self, worker: &WorkerContext) -> Result<(), Error> { + let mut tx = self.pool.begin().await?; + let dead_for = self.config.orphaned_duration().as_secs(); + let queue = self.config.queue.as_ref(); + let count = reenqueue_orphaned(&mut *tx, queue, dead_for).await?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + register_worker(&mut *tx, queue, worker, &Timestamp(now), "PgStorage").await?; + tx.commit().await?; + if count > 0 { + tracing::debug!( + "{count} Re-enqueued orphaned tasks by worker {}", + worker.name() + ); + } + tracing::debug!("Registered Worker: {}", worker.name()); + Ok(()) + } + async fn heartbeat(&mut self, worker: &WorkerContext) -> Result<(), Error> { + let mut txn = self.pool.begin().await?; + let queue = self.config.queue.as_ref(); + let dead_for = self.config.orphaned_duration().as_secs(); + keep_alive(&mut *txn, queue, worker).await?; + let count = reenqueue_orphaned(&mut *txn, queue, dead_for).await?; + txn.commit().await?; + if count > 0 { + tracing::debug!( + "Re-enqueued {count} orphaned tasks by worker {}", + worker.name() + ); + } + Ok(()) + } + async fn fetch_next(&mut self, worker: &WorkerContext) -> Result, Error> { + let mut tx = self.pool.begin().await?; + let res = fetch_next(&mut *tx, &self.config, worker).await?; + tx.commit().await?; + Ok(res) + } + + async fn handle_events( + &mut self, + messages: Vec>, + worker: &WorkerContext, + ) -> Result<(), Error> { + let pool = &self.pool; + let mut lock_ids = messages + .iter() + .filter_map(|msg| { + if let TaskEvent::Lock { task_id, .. } = msg { + Some(task_id.to_string()) + } else { + None + } + }) + .collect::>(); + + let ack_payloads = messages + .iter() + .filter_map(|msg| { + if let TaskEvent::Complete(payload) = msg { + Some(payload) + } else { + None + } + }) + .collect::>(); + + if lock_ids.is_empty() && ack_payloads.is_empty() { + return Ok(()); + } + + tracing::debug!( + "Processing {} messages ({} locks, {} acks)", + messages.len(), + lock_ids.len(), + ack_payloads.len() + ); + + let ack_ids: HashSet = ack_payloads + .iter() + .map(|s| s.task_id().to_string()) + .collect(); + + lock_ids.retain(|id| !ack_ids.contains(id)); + + let mut tx = pool.begin().await?; + + if !ack_payloads.is_empty() { + queries::handle_results(&mut *tx, &ack_payloads, worker.name()).await?; + } + if !lock_ids.is_empty() { + queries::lock_tasks(&mut *tx, &lock_ids, worker.name()).await?; + } + + tx.commit().await?; + Ok(()) + } + + async fn reenqueue_abandoned( + &mut self, + tasks: Vec, + worker: &WorkerContext, + ) -> Result { + let config = &self.config; + let pool = &self.pool; + let mut txn = pool.begin().await?; + let task_ids = tasks + .iter() + .map(|t| t.task_id().unwrap().to_string()) + .collect::>(); + let queue = config.queue.as_ref(); + let count = reenqueue_abandoned(&mut *txn, queue, worker.name(), &task_ids).await?; + if count as usize != tasks.len() { + return Err(Error::ReenqueueMismatch { + queued: tasks.len(), + abandoned: count as usize, + }); + } + txn.commit().await?; + Ok(count) + } + + async fn push_tasks(&mut self, tasks: Vec) -> Result<(), Self::Error> { + let queue = self.config.queue.as_ref(); + let mut tx = self.pool.begin().await?; + push_tasks(&mut *tx, queue, tasks).await?; + tx.commit().await?; + Ok(()) + } +} diff --git a/src/pubsub.rs b/src/pubsub.rs new file mode 100644 index 0000000..87efc0a --- /dev/null +++ b/src/pubsub.rs @@ -0,0 +1,139 @@ +use apalis_core::backend::future::BoxSyncFuture; +use futures::{Stream, StreamExt, TryStreamExt, stream::BoxStream}; +use serde::Deserialize; +use sqlx::postgres::{PgListener, PgNotification}; +use std::{ + pin::Pin, + sync::Mutex, + task::{Context, Poll}, +}; + +use crate::{PgTaskId, error::Error}; + +/// A standalone listener for `apalis::job::insert` +pub struct Pubsub { + state: State, + listener: Option>>>, + pool: sqlx::PgPool, + namespace: String, +} + +impl Clone for Pubsub { + fn clone(&self) -> Self { + Self { + state: State::Starting, + listener: None, + pool: self.pool.clone(), + namespace: self.namespace.clone(), + } + } +} + +impl Pubsub { + pub fn new(pool: sqlx::PgPool, namespace: String) -> Self { + Self { + state: State::Starting, + pool, + namespace, + listener: None, + } + } +} + +enum State { + Starting, + + Connecting { + fut: BoxSyncFuture>, + }, + + Listening, + Closed, +} + +/// A new event emitted when a new job is added +#[derive(Debug, Deserialize)] +pub struct InsertEvent { + pub job_type: String, + pub id: PgTaskId, +} + +impl Stream for Pubsub { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + loop { + match &mut this.state { + State::Starting => { + this.state = State::Connecting { + fut: { + let pool = this.pool.clone(); + let fut = Box::pin(async move { + let mut listener = PgListener::connect_with(&pool).await?; + + listener.listen("apalis::job::insert").await?; + + Ok(listener) + }); + BoxSyncFuture::new(fut) + }, + } + } + State::Connecting { fut } => { + let listener = match fut.poll_unpin(cx) { + Poll::Pending => return Poll::Pending, + + Poll::Ready(Err(err)) => { + this.state = State::Closed; + return Poll::Ready(Some(Err(err))); + } + + Poll::Ready(Ok(listener)) => listener, + }; + + this.listener = Some(Mutex::new( + listener.into_stream().map_err(|e| e.into()).boxed(), + )); + + this.state = State::Listening; + } + + State::Listening => { + let listener = this.listener.as_mut().expect("listener initialized"); + + match listener.get_mut().unwrap().as_mut().poll_next(cx) { + Poll::Pending => return Poll::Pending, + + Poll::Ready(None) => { + this.state = State::Closed; + return Poll::Ready(None); + } + + Poll::Ready(Some(Err(_))) => { + continue; + } + + Poll::Ready(Some(Ok(notification))) => { + let Ok(ev) = + serde_json::from_str::(notification.payload()) + else { + continue; + }; + + if ev.job_type != this.namespace { + continue; + } + + return Poll::Ready(Some(Ok(ev.id))); + } + } + } + State::Closed => { + return Poll::Ready(None); + } + } + } + } +} diff --git a/src/queries/fetch_by_id.rs b/src/queries/fetch_by_id.rs index d9e5afe..5e16122 100644 --- a/src/queries/fetch_by_id.rs +++ b/src/queries/fetch_by_id.rs @@ -1,37 +1,23 @@ -use apalis_core::backend::{BackendExt, FetchById, codec::Codec}; +use apalis_core::backend::{Backend, FetchById}; -use apalis_sql::from_row::{FromRowError, TaskRow}; -use ulid::Ulid; +use crate::{PgTask, PgTaskId, PostgresStorage, error::Error, from_row::PgTaskRow}; -use crate::{CompactType, PgContext, PgTask, PgTaskId, PostgresStorage, from_row::PgTaskRow}; - -impl FetchById for PostgresStorage +impl FetchById for PostgresStorage where - PostgresStorage: - BackendExt, - D: Codec, - D::Error: std::error::Error + Send + Sync + 'static, + Self: Backend, Args: 'static, { fn fetch_by_id( &mut self, id: &PgTaskId, - ) -> impl Future>, Self::Error>> + Send { - let pool = self.pool.clone(); + ) -> impl Future, Self::Error>> + Send { + let pool = self.persistence.pool.clone(); let id = id.to_string(); async move { let task = sqlx::query_file_as!(PgTaskRow, "queries/task/find_by_id.sql", id) .fetch_optional(&pool) .await? - .map(|r: PgTaskRow| { - let row: TaskRow = r.try_into()?; - row.try_into_task_compact() - .and_then(|a| { - a.try_map(|t| D::decode(&t)) - .map_err(|e| FromRowError::DecodeError(e.into())) - }) - .map_err(|e| sqlx::Error::Protocol(e.to_string())) - }) + .map(|r: PgTaskRow| r.try_into()) .transpose()?; Ok(task) } diff --git a/src/queries/fetch_next.rs b/src/queries/fetch_next.rs new file mode 100644 index 0000000..1b6b48c --- /dev/null +++ b/src/queries/fetch_next.rs @@ -0,0 +1,32 @@ +use apalis_core::worker::context::WorkerContext; +use sqlx::Executor; + +use crate::{PgTask, config::Config, error::Error}; + +/// Fetch the next batch of tasks from the sqlite backend +pub async fn fetch_next( + conn: &mut E, + config: &Config, + worker: &WorkerContext, +) -> Result, Error> +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ + use crate::from_row::PgTaskRow; + let job_type = config.queue.as_ref(); + let buffer_size = config.batch_size as i32; + let worker = worker.name(); + + sqlx::query_file_as!( + PgTaskRow, + "queries/task/fetch_next.sql", + worker, + job_type, + buffer_size + ) + .fetch_all(conn) + .await? + .into_iter() + .map(|r| r.try_into()) + .collect() +} diff --git a/src/queries/handle_result.rs b/src/queries/handle_result.rs new file mode 100644 index 0000000..a1b25a7 --- /dev/null +++ b/src/queries/handle_result.rs @@ -0,0 +1,25 @@ +use apalis_core::backend::TaskResult; +use sqlx::Executor; + +use crate::error::Error; + +/// Serialized result payload including the current attempt and status +pub type Payload = TaskResult; + +/// Ack multiple tasks, given a worker +pub async fn handle_results( + executor: &mut E, + results: &[&Payload], + worker_id: &str, +) -> Result +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ + let payload_json = serde_json::to_value(results).map_err(Error::JsonError)?; + + let result = sqlx::query_file!("queries/task/handle_result.sql", payload_json, worker_id) + .execute(executor) + .await?; + + Ok(result.rows_affected()) +} diff --git a/src/queries/keep_alive.rs b/src/queries/keep_alive.rs index a279742..db7534d 100644 --- a/src/queries/keep_alive.rs +++ b/src/queries/keep_alive.rs @@ -1,61 +1,26 @@ use apalis_core::worker::context::WorkerContext; -use apalis_sql::{DateTime, DateTimeExt}; -use futures::{FutureExt, Stream, stream}; -use sqlx::PgPool; +use sqlx::Executor; -use crate::{ - Config, - queries::{ - reenqueue_orphaned::reenqueue_orphaned, register_worker::register as register_worker, - }, -}; +use crate::error::Error; -pub async fn keep_alive( - pool: PgPool, - config: Config, - worker: WorkerContext, -) -> Result<(), sqlx::Error> { - let worker = worker.name().to_owned(); - let queue = config.queue().to_string(); - let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) - .execute(&pool) +/// Heartbeat for denoting liveliness of workers +pub async fn keep_alive(conn: &mut E, queue: &str, worker: &WorkerContext) -> Result<(), Error> +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ + let tasks = worker + .tasks() + .iter() + .map(|task| task.task_id().to_string()) + .collect::>(); + + let worker = worker.name(); + + let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue, &tasks) + .execute(conn) .await?; if res.rows_affected() == 0 { - return Err(sqlx::Error::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - "WORKER_DOES_NOT_EXIST", - ))); + return Err(Error::WorkerOutOfSync); } Ok(()) } - -pub async fn initial_heartbeat( - pool: PgPool, - config: Config, - worker: WorkerContext, - storage_type: &str, -) -> Result<(), sqlx::Error> { - reenqueue_orphaned(pool.clone(), config.clone()).await?; - let last_seen = DateTime::now(); - register_worker( - pool, - config.queue().to_string(), - worker, - last_seen, - storage_type, - ) - .await?; - Ok(()) -} - -pub fn keep_alive_stream( - pool: PgPool, - config: Config, - worker: WorkerContext, -) -> impl Stream> + Send { - stream::unfold((), move |_| { - let register = keep_alive(pool.clone(), config.clone(), worker.clone()); - let interval = apalis_core::timer::Delay::new(*config.keep_alive()); - interval.then(move |_| register.map(|res| Some((res, ())))) - }) -} diff --git a/src/queries/list_queues.rs b/src/queries/list_queues.rs index cd65018..123a9a3 100644 --- a/src/queries/list_queues.rs +++ b/src/queries/list_queues.rs @@ -1,16 +1,14 @@ -use apalis_core::backend::{BackendExt, ListQueues, QueueInfo}; +use apalis_core::backend::{Backend, ListQueues, QueueInfo}; use serde_json::Value; -use ulid::Ulid; -use crate::{CompactType, PgContext, PostgresStorage}; +use crate::{PostgresStorage, error::Error}; -impl ListQueues for PostgresStorage +impl ListQueues for PostgresStorage where - PostgresStorage: - BackendExt, + PostgresStorage: Backend, { fn list_queues(&self) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); + let pool = self.persistence.pool.clone(); struct QueueInfoRow { pub name: Option, pub stats: Option, diff --git a/src/queries/list_tasks.rs b/src/queries/list_tasks.rs index 6c0f919..5ecaca7 100644 --- a/src/queries/list_tasks.rs +++ b/src/queries/list_tasks.rs @@ -1,26 +1,22 @@ use apalis_core::{ - backend::{BackendExt, Filter, ListAllTasks, ListTasks, codec::Codec}, + backend::{Backend, Filter, ListAllTasks, ListTasks}, task::{Task, status::Status}, }; -use apalis_sql::from_row::{FromRowError, TaskRow}; -use ulid::Ulid; -use crate::{CompactType, PgContext, PgTask, PostgresStorage, from_row::PgTaskRow}; +use crate::from_row::PgTaskRow; +use crate::{PgTask, PostgresStorage, error::Error}; -impl ListTasks for PostgresStorage +impl ListTasks for PostgresStorage where - PostgresStorage: - BackendExt, - D: Codec, - D::Error: std::error::Error + Send + Sync + 'static, + PostgresStorage: Backend, Args: 'static, { fn list_tasks( &self, filter: &Filter, - ) -> impl Future>, Self::Error>> + Send { - let queue = self.config.queue().to_string(); - let pool = self.pool.clone(); + ) -> impl Future, Self::Error>> + Send { + let queue = self.persistence.config.queue.to_string(); + let pool = self.persistence.pool.clone(); let limit = filter.limit() as i64; let offset = filter.offset() as i64; let status = filter @@ -40,38 +36,27 @@ where .fetch_all(&pool) .await? .into_iter() - .map(|r| { - let row: TaskRow = r.try_into()?; - row.try_into_task_compact() - .and_then(|a| { - a.try_map(|t| D::decode(&t)) - .map_err(|e| FromRowError::DecodeError(e.into())) - }) - .map_err(|e| sqlx::Error::Protocol(e.to_string())) - }) + .map(|r| r.try_into()) .collect::, _>>()?; Ok(tasks) } } } -impl ListAllTasks for PostgresStorage +impl ListAllTasks for PostgresStorage where - PostgresStorage: - BackendExt, + PostgresStorage: Backend, { fn list_all_tasks( &self, filter: &Filter, - ) -> impl Future< - Output = Result>, Self::Error>, - > + Send { + ) -> impl Future>, Self::Error>> + Send { let status = filter .status .as_ref() .map(|s| s.to_string()) .unwrap_or(Status::Pending.to_string()); - let pool = self.pool.clone(); + let pool = self.persistence.pool.clone(); let limit = filter.limit() as i64; let offset = filter.offset() as i64; async move { @@ -85,11 +70,7 @@ where .fetch_all(&pool) .await? .into_iter() - .map(|r| { - let row: TaskRow = r.try_into()?; - row.try_into_task_compact() - .map_err(|e| sqlx::Error::Protocol(e.to_string())) - }) + .map(|r| r.try_into()) .collect::, _>>()?; Ok(tasks) } diff --git a/src/queries/list_workers.rs b/src/queries/list_workers.rs index ca2feaf..8f44a4e 100644 --- a/src/queries/list_workers.rs +++ b/src/queries/list_workers.rs @@ -1,7 +1,8 @@ -use apalis_core::backend::{BackendExt, ListWorkers, RunningWorker}; -use apalis_sql::{DateTime, DateTimeExt}; +use apalis_core::backend::{Backend, ListWorkers, RunningWorker}; + use futures::TryFutureExt; -use ulid::Ulid; + +use crate::{PostgresStorage, error::Error, timestamp::Timestamp}; #[derive(Debug)] pub struct WorkerRow { @@ -9,21 +10,18 @@ pub struct WorkerRow { pub worker_type: String, pub storage_name: String, pub layers: Option, - pub last_seen: DateTime, - pub started_at: Option, + pub last_seen: Timestamp, + pub started_at: Option, } -use crate::{CompactType, PgContext, PostgresStorage}; - -impl ListWorkers for PostgresStorage +impl ListWorkers for PostgresStorage where - PostgresStorage: - BackendExt, + PostgresStorage: Backend, { fn list_workers(&self) -> impl Future, Self::Error>> + Send { - let queue = self.config.queue().to_string(); + let queue = self.persistence.config.queue.to_string(); - let pool = self.pool.clone(); + let pool = self.persistence.pool.clone(); let limit = 100; let offset = 0; async move { @@ -40,11 +38,8 @@ where .map(|w| RunningWorker { id: w.id, backend: w.storage_name, - started_at: w - .started_at - .map(|t| t.to_unix_timestamp()) - .unwrap_or_default() as u64, - last_heartbeat: w.last_seen.to_unix_timestamp() as u64, + started_at: w.started_at.unwrap_or_default().0, + last_heartbeat: w.last_seen.0, layers: w.layers.unwrap_or_default(), queue: w.worker_type, }) @@ -58,7 +53,7 @@ where fn list_all_workers( &self, ) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); + let pool = self.persistence.pool.clone(); let limit = 100; let offset = 0; async move { @@ -74,11 +69,8 @@ where .map(|w| RunningWorker { id: w.id, backend: w.storage_name, - started_at: w - .started_at - .map(|t| t.to_unix_timestamp()) - .unwrap_or_default() as u64, - last_heartbeat: w.last_seen.to_unix_timestamp() as u64, + started_at: w.started_at.unwrap_or_default().0, + last_heartbeat: w.last_seen.0, layers: w.layers.unwrap_or_default(), queue: w.worker_type, }) diff --git a/src/queries/lock_task.rs b/src/queries/lock_task.rs new file mode 100644 index 0000000..5b73222 --- /dev/null +++ b/src/queries/lock_task.rs @@ -0,0 +1,13 @@ +use sqlx::{Error, Executor}; + +/// Lock multiple tasks, given a worker +pub async fn lock_tasks(conn: &mut E, task_ids: &[String], worker_id: &str) -> Result +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ + let res = sqlx::query_file!("queries/task/lock_by_id.sql", task_ids, worker_id) + .execute(&mut *conn) + .await?; + + Ok(res.rows_affected()) +} diff --git a/src/queries/metrics.rs b/src/queries/metrics.rs index 689abac..d1beda1 100644 --- a/src/queries/metrics.rs +++ b/src/queries/metrics.rs @@ -1,7 +1,8 @@ -use apalis_core::backend::{BackendExt, Metrics, Statistic}; -use ulid::Ulid; +use std::str::FromStr; -use crate::{CompactType, PgContext, PostgresStorage}; +use apalis_core::backend::{Backend, Metrics, StatType, Statistic}; + +use crate::{PostgresStorage, error::Error}; struct StatisticRow { priority: Option, @@ -10,13 +11,12 @@ struct StatisticRow { value: Option, } -impl Metrics for PostgresStorage +impl Metrics for PostgresStorage where - PostgresStorage: - BackendExt, + PostgresStorage: Backend, { fn global(&self) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); + let pool = self.persistence.pool.clone(); async move { let rec = sqlx::query_file_as!(StatisticRow, "queries/backend/overview.sql") @@ -25,7 +25,8 @@ where .into_iter() .map(|r| Statistic { priority: Some(r.priority.unwrap_or_default() as u64), - stat_type: apalis_sql::stat_type_from_string(&r.r#type.unwrap_or_default()), + stat_type: StatType::from_str(&r.r#type.unwrap_or_default()) + .unwrap_or_default(), title: r.statistic.unwrap_or_default(), value: r.value.unwrap_or_default().to_string(), }) @@ -34,8 +35,8 @@ where } } fn fetch_by_queue(&self) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - let queue_id = self.config.queue().to_string(); + let pool = self.persistence.pool.clone(); + let queue_id = self.persistence.config.queue.to_string(); async move { let rec = sqlx::query_file_as!( StatisticRow, @@ -47,7 +48,7 @@ where .into_iter() .map(|r| Statistic { priority: Some(r.priority.unwrap_or_default() as u64), - stat_type: apalis_sql::stat_type_from_string(&r.r#type.unwrap_or_default()), + stat_type: StatType::from_str(&r.r#type.unwrap_or_default()).unwrap_or_default(), title: r.statistic.unwrap_or_default(), value: r.value.unwrap_or_default().to_string(), }) diff --git a/src/queries/mod.rs b/src/queries/mod.rs index aab99c3..60017bd 100644 --- a/src/queries/mod.rs +++ b/src/queries/mod.rs @@ -1,9 +1,20 @@ -pub mod fetch_by_id; -pub mod keep_alive; -pub mod list_queues; -pub mod list_tasks; -pub mod list_workers; -pub mod metrics; -pub mod reenqueue_orphaned; -pub mod register_worker; -pub mod wait_for; +//! Queries needed for polling, updating and exposing tasks. +mod fetch_by_id; +mod fetch_next; +mod handle_result; +mod keep_alive; +mod list_queues; +mod list_tasks; +mod list_workers; +mod lock_task; +mod metrics; +mod reenqueue_orphaned; +mod register_worker; +mod wait_for; + +pub use crate::queries::{ + fetch_next::fetch_next, handle_result::Payload as ResultPayload, handle_result::handle_results, + keep_alive::keep_alive, lock_task::lock_tasks, reenqueue_orphaned::reenqueue_abandoned, + reenqueue_orphaned::reenqueue_orphaned, register_worker::register_worker, +}; +pub use crate::sink::push_tasks; diff --git a/src/queries/reenqueue_orphaned.rs b/src/queries/reenqueue_orphaned.rs index 2c3b012..7ee480f 100644 --- a/src/queries/reenqueue_orphaned.rs +++ b/src/queries/reenqueue_orphaned.rs @@ -1,57 +1,57 @@ -use std::time::Duration; +use futures::TryFutureExt; +use sqlx::{Executor, postgres::types::PgInterval}; -use futures::{FutureExt, Stream, stream}; -use sqlx::{PgPool, postgres::types::PgInterval}; +use crate::error::Error; -use crate::Config; - -pub fn reenqueue_orphaned( - pool: PgPool, - config: Config, -) -> impl Future> + Send { - let dead_for = config.reenqueue_orphaned_after().as_secs() as i64; - let queue = config.queue().to_string(); +/// Reenqueue jobs orphaned by a dead worker +pub async fn reenqueue_orphaned(conn: &mut E, queue: &str, dead_for: u64) -> Result +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ let dead_for = PgInterval { months: 0, days: 0, - microseconds: dead_for * 1_000_000, + microseconds: dead_for as i64 * 1_000_000, }; - async move { - match sqlx::query_file!("queries/backend/reenqueue_orphaned.sql", dead_for, queue,) - .execute(&pool) - .await - { - Ok(res) => { - if res.rows_affected() > 0 { - // log::info!( - // "Re-enqueued {} orphaned tasks that were being processed by dead workers", - // res.rows_affected() - // ); - } - Ok(res.rows_affected()) - } - Err(e) => { - // log::error!("Failed to re-enqueue orphaned tasks: {e}"); - Err(e) + + match sqlx::query_file!("queries/backend/reenqueue_orphaned.sql", dead_for, queue,) + .execute(conn) + .await + { + Ok(res) => { + if res.rows_affected() > 0 { + tracing::info!( + "Re-enqueued {} orphaned tasks that were being processed by dead workers", + res.rows_affected() + ); } + Ok(res.rows_affected()) + } + Err(e) => { + tracing::error!("Failed to re-enqueue orphaned tasks: {e}"); + Err(e.into()) } } } -pub fn reenqueue_orphaned_stream( - pool: PgPool, - config: Config, - interval: Duration, -) -> impl Stream> + Send { - let config = config.clone(); - stream::unfold((), move |_| { - let pool = pool.clone(); - let config = config.clone(); - let interval = apalis_core::timer::Delay::new(interval); - let fut = async move { - interval.await; - reenqueue_orphaned(pool, config).await - }; - fut.map(|res| Some((res, ()))) - }) +/// Rescues tasks that could not be executed after a worker shutdown +pub async fn reenqueue_abandoned( + executor: &mut E, + queue: &str, + worker: &str, + task_ids: &[String], +) -> Result +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ + let res = sqlx::query_file!( + "queries/worker/reenqueue_abandoned.sql", + queue, + worker, + task_ids + ) + .execute(executor) + .map_ok(|res| res.rows_affected()) + .await?; + Ok(res) } diff --git a/src/queries/register_worker.rs b/src/queries/register_worker.rs index 695cee8..18e5602 100644 --- a/src/queries/register_worker.rs +++ b/src/queries/register_worker.rs @@ -1,29 +1,33 @@ use apalis_core::worker::context::WorkerContext; -use apalis_sql::DateTime; -use sqlx::PgPool; +use sqlx::Executor; -pub async fn register( - pool: PgPool, - worker_type: String, - worker: WorkerContext, - last_seen: DateTime, +use crate::{error::Error, timestamp::Timestamp}; + +/// Register a worker in the database +/// +/// Errors if worker already exists +pub async fn register_worker( + conn: &mut E, + queue: &str, + worker: &WorkerContext, + last_seen: &Timestamp, backend_type: &str, -) -> Result<(), sqlx::Error> { +) -> Result<(), Error> +where + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres>, +{ let res = sqlx::query_file!( "queries/worker/register.sql", worker.name(), - worker_type, + queue, backend_type, worker.get_service(), last_seen ) - .execute(&pool) + .execute(conn) .await?; if res.rows_affected() == 0 { - return Err(sqlx::Error::Io(std::io::Error::new( - std::io::ErrorKind::AddrInUse, - "WORKER_ALREADY_EXISTS", - ))); + return Err(Error::WorkerAlreadyExists(worker.name().to_owned())); } Ok(()) } diff --git a/src/queries/wait_for.rs b/src/queries/wait_for.rs index 4c9b707..b15ff3f 100644 --- a/src/queries/wait_for.rs +++ b/src/queries/wait_for.rs @@ -1,37 +1,34 @@ use std::{collections::HashSet, str::FromStr, vec}; use apalis_core::{ - backend::{BackendExt, TaskResult, WaitForCompletion}, - task::{status::Status, task_id::TaskId}, + backend::{Backend, TaskResult, WaitForCompletion}, + task::{ + status::{Status, StatusError}, + task_id::TaskId, + }, }; use futures::{StreamExt, stream::BoxStream}; use serde::de::DeserializeOwned; -use ulid::Ulid; -use crate::{CompactType, PgContext, PostgresStorage}; +use crate::{PostgresStorage, error::Error}; #[derive(Debug)] pub struct TaskResultRow { pub id: Option, pub status: Option, pub result: Option, + pub attempt: Option, } -impl WaitForCompletion - for PostgresStorage +impl WaitForCompletion for PostgresStorage where - PostgresStorage: - BackendExt, + PostgresStorage: Backend, Result: DeserializeOwned, { - type ResultStream = BoxStream<'static, Result, Self::Error>>; - fn wait_for( - &self, - task_ids: impl IntoIterator>, - ) -> Self::ResultStream { - let pool = self.pool.clone(); + type ResultStream = BoxStream<'static, Result, Self::Error>>; + fn wait_for(&self, task_ids: impl IntoIterator) -> Self::ResultStream { let ids: HashSet = task_ids.into_iter().map(|id| id.to_string()).collect(); - + let pool = self.persistence.pool.clone(); let stream = futures::stream::unfold(ids, move |mut remaining_ids| { let pool = pool.clone(); async move { @@ -59,15 +56,14 @@ where for row in rows { let task_id = row.id.clone().unwrap(); remaining_ids.remove(&task_id); - // Here we would normally decode the output O from the row - // For simplicity, we assume O is String and the output is stored in row.output let result: Result = serde_json::from_value(row.result.unwrap()).unwrap(); - results.push(Ok(TaskResult::new( - TaskId::from_str(&task_id).ok()?, - Status::from_str(&row.status.unwrap()).ok()?, + results.push(Ok(TaskResult { + task_id: TaskId::from_str(&task_id).ok()?, + status: Status::from_str(&row.status.unwrap()).ok()?, + attempt: row.attempt.unwrap_or_default() as usize, result, - ))); + })); } Some((futures::stream::iter(results), remaining_ids)) @@ -79,13 +75,13 @@ where // Implementation of check_status fn check_status( &self, - task_ids: impl IntoIterator> + Send, - ) -> impl Future>, Self::Error>> + Send { - let pool = self.pool.clone(); + task_ids: impl IntoIterator + Send, + ) -> impl Future>, Self::Error>> + Send { + let pool = self.persistence.pool.clone(); let ids: Vec = task_ids.into_iter().map(|id| id.to_string()).collect(); async move { - let ids = serde_json::to_value(&ids).unwrap(); + let ids = serde_json::to_value(&ids).map_err(Error::JsonError)?; let rows = sqlx::query_file_as!( TaskResultRow, "queries/backend/fetch_completed_tasks.sql", @@ -96,20 +92,21 @@ where let mut results = Vec::new(); for row in rows { - let task_id = TaskId::from_str(&row.id.unwrap()) - .map_err(|_| sqlx::Error::Protocol("Invalid task ID".into()))?; + let task_id = TaskId::from_str(&row.id.unwrap()).map_err(Error::TaskIdError)?; - let result: Result = serde_json::from_value(row.result.unwrap()) - .map_err(|_| sqlx::Error::Protocol("Failed to decode result".into()))?; + let result: Result = + serde_json::from_value(row.result.unwrap()).map_err(Error::JsonError)?; - results.push(TaskResult::new( + results.push(TaskResult { task_id, - row.status + status: row + .status .unwrap() .parse() - .map_err(|_| sqlx::Error::Protocol("Invalid status value".into()))?, + .map_err(|e: StatusError| Error::StatusError(e))?, result, - )); + attempt: 0, // attempt: row.attempt.unwrap_or_default() as usize, + }); } Ok(results) diff --git a/src/shared.rs b/src/shared.rs deleted file mode 100644 index 46a08a5..0000000 --- a/src/shared.rs +++ /dev/null @@ -1,339 +0,0 @@ -use std::{ - collections::HashMap, - future::ready, - marker::PhantomData, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -use crate::{ - CompactType, Config, InsertEvent, PgContext, PgTask, PgTaskId, PostgresStorage, - ack::{LockTaskLayer, PgAck}, - fetcher::PgPollFetcher, - queries::{ - keep_alive::{initial_heartbeat, keep_alive_stream}, - reenqueue_orphaned::reenqueue_orphaned_stream, - }, -}; -use crate::{from_row::PgTaskRow, sink::PgSink}; -use apalis_codec::json::JsonCodec; -use apalis_core::{ - backend::{Backend, BackendExt, TaskStream, codec::Codec, queue::Queue, shared::MakeShared}, - layers::Stack, - worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, -}; -use apalis_sql::from_row::TaskRow; -use futures::{ - FutureExt, SinkExt, Stream, StreamExt, TryFutureExt, TryStreamExt, - channel::mpsc::{self, Receiver, Sender}, - future::{BoxFuture, Shared}, - lock::Mutex, - stream::{self, BoxStream, select}, -}; -use sqlx::{PgPool, postgres::PgListener}; -use ulid::Ulid; - -pub struct SharedPostgresStorage> { - pool: PgPool, - registry: Arc>>>, - drive: Shared>, - _marker: PhantomData<(Compact, Codec)>, -} - -impl SharedPostgresStorage { - pub fn new(pool: PgPool) -> Self { - let registry: Arc>>> = - Arc::new(Mutex::new(HashMap::default())); - let p = pool.clone(); - let instances = registry.clone(); - Self { - pool, - drive: async move { - let mut listener = PgListener::connect_with(&p).await.unwrap(); - listener.listen("apalis::job::insert").await.unwrap(); - listener - .into_stream() - .filter_map(|notification| { - let instances = instances.clone(); - async move { - let pg_notification = notification.ok()?; - let payload = pg_notification.payload(); - let ev: InsertEvent = serde_json::from_str(payload).ok()?; - let instances = instances.lock().await; - if instances.get(&ev.job_type).is_some() { - return Some(ev); - } - None - } - }) - .for_each(|ev| { - let instances = instances.clone(); - async move { - let mut instances = instances.lock().await; - let sender = instances.get_mut(&ev.job_type).unwrap(); - sender.send(ev.id).await.unwrap(); - } - }) - .await; - } - .boxed() - .shared(), - registry, - _marker: PhantomData, - } - } -} -#[derive(Debug, thiserror::Error)] -pub enum SharedPostgresError { - /// Namespace not found - #[error("namespace already exists: {0}")] - NamespaceExists(String), - - /// Registry locked - #[error("registry locked")] - RegistryLocked, -} - -impl MakeShared for SharedPostgresStorage { - type Backend = PostgresStorage; - type Config = Config; - type MakeError = SharedPostgresError; - fn make_shared(&mut self) -> Result - where - Self::Config: Default, - { - Self::make_shared_with_config(self, Config::new(std::any::type_name::())) - } - fn make_shared_with_config( - &mut self, - config: Self::Config, - ) -> Result { - let (tx, rx) = mpsc::channel(config.buffer_size()); - let mut r = self - .registry - .try_lock() - .ok_or(SharedPostgresError::RegistryLocked)?; - if r.insert(config.queue().to_string(), tx).is_some() { - return Err(SharedPostgresError::NamespaceExists( - config.queue().to_string(), - )); - } - let sink = PgSink::new(&self.pool, &config); - Ok(PostgresStorage { - _marker: PhantomData, - config, - fetcher: SharedFetcher { - poller: self.drive.clone(), - receiver: Arc::new(Mutex::new(rx)), - }, - pool: self.pool.clone(), - sink, - }) - } -} - -#[derive(Clone, Debug)] -pub struct SharedFetcher { - poller: Shared>, - receiver: Arc>>, -} - -impl Stream for SharedFetcher { - type Item = PgTaskId; - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - // Keep the poller alive by polling it, but ignoring the output - let _ = this.poller.poll_unpin(cx); - - // Delegate actual items to receiver - let mut receiver = this.receiver.try_lock(); - if let Some(ref mut rx) = receiver { - rx.poll_next_unpin(cx) - } else { - Poll::Pending - } - } -} - -impl Backend for PostgresStorage -where - Args: Send + 'static + Unpin, - Decode: Codec + 'static + Unpin + Send, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Args = Args; - - type IdType = Ulid; - - type Error = sqlx::Error; - - type Stream = TaskStream, Self::Error>; - - type Beat = BoxStream<'static, Result<(), Self::Error>>; - - type Context = PgContext; - - type Layer = Stack>; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let pool = self.pool.clone(); - let config = self.config.clone(); - let worker = worker.clone(); - let keep_alive = keep_alive_stream(pool, config, worker); - let reenqueue = reenqueue_orphaned_stream( - self.pool.clone(), - self.config.clone(), - *self.config.keep_alive(), - ) - .map_ok(|_| ()); - futures::stream::select(keep_alive, reenqueue).boxed() - } - - fn middleware(&self) -> Self::Layer { - Stack::new( - LockTaskLayer::new(self.pool.clone()), - AcknowledgeLayer::new(PgAck::new(self.pool.clone())), - ) - } - - fn poll(self, worker: &WorkerContext) -> Self::Stream { - self.poll_shared(worker) - .map(|a| match a { - Ok(Some(task)) => Ok(Some( - task.try_map(|t| Decode::decode(&t)) - .map_err(|e| sqlx::Error::Decode(e.into()))?, - )), - Ok(None) => Ok(None), - Err(e) => Err(e), - }) - .boxed() - } -} - -impl BackendExt for PostgresStorage -where - Args: Send + 'static + Unpin, - Decode: Codec + 'static + Unpin + Send, - Decode::Error: std::error::Error + Send + Sync + 'static, -{ - type Compact = CompactType; - - type Codec = Decode; - type CompactStream = TaskStream, Self::Error>; - - fn get_queue(&self) -> Queue { - self.config.queue().clone() - } - - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream { - self.poll_shared(worker).boxed() - } -} - -impl PostgresStorage { - fn poll_shared( - self, - worker: &WorkerContext, - ) -> impl Stream>, sqlx::Error>> + 'static { - let pool = self.pool.clone(); - let worker_id = worker.name().to_owned(); - let register_worker = initial_heartbeat( - self.pool.clone(), - self.config.clone(), - worker.clone(), - "SharedPostgresStorage", - ) - .map_ok(|_| None); - let register = stream::once(register_worker); - let lazy_fetcher = self - .fetcher - .map(|t| t.to_string()) - .ready_chunks(self.config.buffer_size()) - .then(move |ids| { - let pool = pool.clone(); - let worker_id = worker_id.clone(); - async move { - let mut tx = pool.begin().await?; - let res: Vec<_> = sqlx::query_file_as!( - PgTaskRow, - "queries/task/queue_by_id.sql", - &ids, - &worker_id - ) - .fetch(&mut *tx) - .map(|r| { - let row: TaskRow = r?.try_into()?; - Ok(Some( - row.try_into_task_compact() - .map_err(|e| sqlx::Error::Protocol(e.to_string()))?, - )) - }) - .collect() - .await; - tx.commit().await?; - Ok::<_, sqlx::Error>(res) - } - }) - .flat_map(|vec| match vec { - Ok(vec) => stream::iter(vec.into_iter().map(|res| match res { - Ok(t) => Ok(t), - Err(e) => Err(e), - })) - .boxed(), - Err(e) => stream::once(ready(Err(e))).boxed(), - }) - .boxed(); - let eager_fetcher = StreamExt::boxed(PgPollFetcher::::new( - &self.pool, - &self.config, - worker, - )); - register.chain(select(lazy_fetcher, eager_fetcher)) - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use apalis_core::{backend::TaskSink, error::BoxDynError, worker::builder::WorkerBuilder}; - - use super::*; - - #[tokio::test] - async fn basic_worker() { - let pool = PgPool::connect(std::env::var("DATABASE_URL").unwrap().as_str()) - .await - .unwrap(); - let mut store = SharedPostgresStorage::new(pool); - - let mut map_store = store.make_shared().unwrap(); - - let mut int_store = store.make_shared().unwrap(); - - map_store - .push_stream(&mut stream::iter(vec![HashMap::::new()])) - .await - .unwrap(); - int_store.push(99).await.unwrap(); - - async fn send_reminder( - _: T, - _task_id: PgTaskId, - wrk: WorkerContext, - ) -> Result<(), BoxDynError> { - tokio::time::sleep(Duration::from_secs(2)).await; - wrk.stop().unwrap(); - Ok(()) - } - - let int_worker = WorkerBuilder::new("rango-tango-2") - .backend(int_store) - .build(send_reminder); - let map_worker = WorkerBuilder::new("rango-tango-1") - .backend(map_store) - .build(send_reminder); - tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); - } -} diff --git a/src/sink.rs b/src/sink.rs index cfd5f01..16116a7 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -1,52 +1,22 @@ -use apalis_codec::json::JsonCodec; -use apalis_sql::{DateTime, DateTimeExt, config::Config}; -use futures::{ - FutureExt, Sink, TryFutureExt, - future::{BoxFuture, Shared}, -}; -use sqlx::{Executor, PgPool}; +use futures::{FutureExt, Sink, TryFutureExt}; +use sqlx::{Executor, postgres::types::PgHstore}; use std::{ pin::Pin, - sync::Arc, task::{Context, Poll}, }; use ulid::Ulid; -use crate::{CompactType, PgTask, PostgresStorage}; - -type FlushFuture = BoxFuture<'static, Result<(), Arc>>; +use crate::{PgTask, backend::PostgresStorage, error::Error, timestamp::Timestamp}; -#[pin_project::pin_project] -pub struct PgSink> { - pool: PgPool, - config: Config, - buffer: Vec>, - #[pin] - flush_future: Option>, - _marker: std::marker::PhantomData<(Args, Codec)>, -} - -impl Clone for PgSink { - fn clone(&self) -> Self { - Self { - pool: self.pool.clone(), - config: self.config.clone(), - buffer: Vec::new(), - flush_future: None, - _marker: std::marker::PhantomData, - } - } -} - -pub fn push_tasks<'a, E>( - conn: E, - cfg: Config, - buffer: Vec>, -) -> impl futures::Future> + Send + 'a +/// Push a batch of tasks to the database +pub fn push_tasks( + conn: &mut E, + queue: &str, + buffer: Vec, +) -> impl futures::Future> + Send where - E: Executor<'a, Database = sqlx::Postgres> + Send + 'a, + for<'e> &'e mut E: Executor<'e, Database = sqlx::Postgres> + Send, { - let job_type = cfg.queue().to_string(); // Build the multi-row INSERT with UNNEST let mut ids = Vec::new(); let mut job_data = Vec::new(); @@ -55,28 +25,33 @@ where let mut max_attempts_vec = Vec::new(); let mut metadata = Vec::new(); let mut idempotency_key: Vec> = Vec::new(); - + let now = Timestamp::now(); for task in buffer { ids.push( - task.parts - .task_id + task.task_id() .map(|id| id.to_string()) - .unwrap_or(Ulid::new().to_string()), + .unwrap_or(Ulid::generate().to_string()), ); - job_data.push(task.args); - run_ats.push(::from_unix_timestamp( - task.parts.run_at as i64, + + run_ats.push(task.run_at().map(|f| f as i64).unwrap_or(now.0 as i64)); + priorities.push(task.priority().map(|f| f as i32).unwrap_or_default()); + max_attempts_vec.push(task.max_attempts().map(|f| f as i32).unwrap_or(25)); + metadata.push(PgHstore( + task.metadata() + .clone() + .into_inner() + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect(), )); - priorities.push(task.parts.ctx.priority()); - max_attempts_vec.push(task.parts.ctx.max_attempts()); - metadata.push(serde_json::Value::Object(task.parts.ctx.meta().clone())); - idempotency_key.push(task.parts.idempotency_key); + idempotency_key.push(task.idempotency_key().map(|a| a.to_owned())); + job_data.push(task.args); } sqlx::query_file!( "queries/task/sink.sql", &ids, - &job_type, + &queue, &job_data, &max_attempts_vec, &run_ats, @@ -86,87 +61,29 @@ where ) .execute(conn) .map_ok(|_| ()) + .map_err(|e| e.into()) .boxed() } -impl PgSink { - pub fn new(pool: &PgPool, config: &Config) -> Self { - Self { - pool: pool.clone(), - config: config.clone(), - buffer: Vec::new(), - _marker: std::marker::PhantomData, - flush_future: None, - } - } -} - -impl Sink> - for PostgresStorage +impl Sink for PostgresStorage where Args: Unpin + Send + Sync + 'static, - Fetcher: Unpin, { - type Error = sqlx::Error; + type Error = Error; - fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().persistence.poll_ready(cx) } - fn start_send(self: Pin<&mut Self>, item: PgTask) -> Result<(), Self::Error> { - // Add the item to the buffer - self.get_mut().sink.buffer.push(item); - Ok(()) + fn start_send(self: Pin<&mut Self>, item: PgTask) -> Result<(), Self::Error> { + self.project().persistence.start_send(item) } fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - // If there's no existing future and buffer is empty, we're done - if this.sink.flush_future.is_none() && this.sink.buffer.is_empty() { - return Poll::Ready(Ok(())); - } - - // Create the future only if we don't have one and there's work to do - if this.sink.flush_future.is_none() && !this.sink.buffer.is_empty() { - let config = this.config.clone(); - let buffer = std::mem::take(&mut this.sink.buffer); - let pool = this.sink.pool.clone(); - let fut = async move { - let mut conn = pool.begin().map_err(Arc::new).await?; - push_tasks(&mut *conn, config, buffer) - .map_err(Arc::new) - .await?; - conn.commit().map_err(Arc::new).await?; - Ok(()) - }; - this.sink.flush_future = Some(fut.boxed().shared()); - } - - // Poll the existing future - if let Some(mut fut) = this.sink.flush_future.take() { - match fut.poll_unpin(cx) { - Poll::Ready(Ok(())) => { - // Future completed successfully, don't put it back - Poll::Ready(Ok(())) - } - Poll::Ready(Err(e)) => { - // Future completed with error, don't put it back - Poll::Ready(Err(Arc::::into_inner(e).unwrap())) - } - Poll::Pending => { - // Future is still pending, put it back and return Pending - this.sink.flush_future = Some(fut); - Poll::Pending - } - } - } else { - // No future and no work to do - Poll::Ready(Ok(())) - } + self.project().persistence.poll_flush(cx) } fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.poll_flush(cx) + Sink::poll_close(self.project().persistence, cx) } } diff --git a/src/timestamp.rs b/src/timestamp.rs new file mode 100644 index 0000000..adf8d13 --- /dev/null +++ b/src/timestamp.rs @@ -0,0 +1,71 @@ +use serde::{Deserialize, Serialize}; +use sqlx::{ + decode::Decode, + encode::{Encode, IsNull}, + error::BoxDynError, + postgres::{PgArgumentBuffer, PgHasArrayType, PgTypeInfo, PgValueRef, Postgres, types::Oid}, + types::Type, +}; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct Timestamp(pub u64); + +const POSTGRES_EPOCH: i64 = 946_684_800; + +impl Type for Timestamp { + fn type_info() -> PgTypeInfo { + PgTypeInfo::with_oid(Oid(1184)) + } +} + +impl PgHasArrayType for Timestamp { + fn array_type_info() -> PgTypeInfo { + PgTypeInfo::with_oid(Oid(1185)) + } +} + +impl<'q> Encode<'q, Postgres> for Timestamp { + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> Result { + // PostgreSQL stores TIMESTAMPTZ as microseconds + // since 2000-01-01 00:00:00 UTC. + let postgres_micros = (self.0 as i64 - POSTGRES_EPOCH) + .checked_mul(1_000_000) + .ok_or("timestamp overflow")?; + + >::encode(postgres_micros, buf) + } + + fn size_hint(&self) -> usize { + 8 + } +} + +impl<'r> Decode<'r, Postgres> for Timestamp { + fn decode(value: PgValueRef<'r>) -> Result { + // PostgreSQL TIMESTAMPTZ -> microseconds since 2000. + let postgres_micros = >::decode(value)?; + + // Convert to Unix seconds. + let unix_seconds = postgres_micros / 1_000_000 + POSTGRES_EPOCH; + + if unix_seconds < 0 { + return Err("timestamp is before Unix epoch".into()); + } + + Ok(Timestamp(unix_seconds as u64)) + } +} + +impl From for Timestamp { + fn from(num: i64) -> Self { + Self(num as u64) + } +} + +impl Timestamp { + pub fn now() -> Self { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("System time is before Unix epoch"); + Timestamp(now.as_secs()) + } +} diff --git a/supply-chain/config.toml b/supply-chain/config.toml index cdc73c9..73f3853 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -7,40 +7,32 @@ version = "0.10" [policy.apalis-postgres] audit-as-crates-io = true +[[exemptions.aho-corasick]] +version = "1.1.5" +criteria = "safe-to-run" + [[exemptions.allocator-api2]] version = "0.2.21" criteria = "safe-to-deploy" -[[exemptions.android_system_properties]] -version = "0.1.5" -criteria = "safe-to-deploy" - -[[exemptions.anyhow]] -version = "1.0.102" -criteria = "safe-to-deploy" - [[exemptions.apalis]] -version = "1.0.0-rc.9" +version = "1.0.0-rc.10" criteria = "safe-to-run" [[exemptions.apalis-codec]] -version = "0.1.0-rc.9" +version = "0.1.0-rc.10" criteria = "safe-to-deploy" [[exemptions.apalis-core]] -version = "1.0.0-rc.9" +version = "1.0.0-rc.10" criteria = "safe-to-deploy" [[exemptions.apalis-postgres]] version = "1.0.0-beta.3" criteria = "safe-to-deploy" -[[exemptions.apalis-sql]] -version = "1.0.0-rc.9" -criteria = "safe-to-deploy" - [[exemptions.apalis-workflow]] -version = "0.1.0-rc.9" +version = "0.1.0-rc.10" criteria = "safe-to-run" [[exemptions.async-channel]] @@ -88,7 +80,7 @@ version = "1.1.2" criteria = "safe-to-deploy" [[exemptions.autocfg]] -version = "1.5.0" +version = "1.5.1" criteria = "safe-to-deploy" [[exemptions.base64]] @@ -96,7 +88,7 @@ version = "0.22.1" criteria = "safe-to-deploy" [[exemptions.bitflags]] -version = "2.11.1" +version = "2.13.2" criteria = "safe-to-deploy" [[exemptions.block-buffer]] @@ -104,15 +96,15 @@ version = "0.10.4" criteria = "safe-to-deploy" [[exemptions.block-buffer]] -version = "0.12.0" +version = "0.12.1" criteria = "safe-to-deploy" [[exemptions.blocking]] -version = "1.6.2" +version = "1.7.0" criteria = "safe-to-deploy" [[exemptions.bumpalo]] -version = "3.20.2" +version = "3.20.3" criteria = "safe-to-deploy" [[exemptions.byteorder]] @@ -120,23 +112,19 @@ version = "1.5.0" criteria = "safe-to-deploy" [[exemptions.bytes]] -version = "1.11.1" +version = "1.12.1" criteria = "safe-to-deploy" [[exemptions.cc]] -version = "1.2.61" +version = "1.4.6" criteria = "safe-to-deploy" [[exemptions.cfg-if]] -version = "1.0.4" +version = "1.0.5" criteria = "safe-to-deploy" [[exemptions.chacha20]] -version = "0.10.0" -criteria = "safe-to-deploy" - -[[exemptions.chrono]] -version = "0.4.44" +version = "0.10.2" criteria = "safe-to-deploy" [[exemptions.cmov]] @@ -160,7 +148,7 @@ version = "0.2.17" criteria = "safe-to-deploy" [[exemptions.cpufeatures]] -version = "0.3.0" +version = "0.3.1" criteria = "safe-to-deploy" [[exemptions.crc]] @@ -172,15 +160,15 @@ version = "2.5.0" criteria = "safe-to-deploy" [[exemptions.crossbeam-queue]] -version = "0.3.12" +version = "0.3.14" criteria = "safe-to-deploy" [[exemptions.crossbeam-utils]] -version = "0.8.21" +version = "0.8.23" criteria = "safe-to-deploy" [[exemptions.crypto-common]] -version = "0.1.7" +version = "0.1.6" criteria = "safe-to-deploy" [[exemptions.crypto-common]] @@ -191,8 +179,8 @@ criteria = "safe-to-deploy" version = "0.4.2" criteria = "safe-to-deploy" -[[exemptions.deranged]] -version = "0.5.8" +[[exemptions.dashmap]] +version = "6.2.1" criteria = "safe-to-deploy" [[exemptions.digest]] @@ -204,7 +192,7 @@ version = "0.11.3" criteria = "safe-to-deploy" [[exemptions.displaydoc]] -version = "0.2.5" +version = "0.2.7" criteria = "safe-to-deploy" [[exemptions.dotenvy]] @@ -212,7 +200,7 @@ version = "0.15.7" criteria = "safe-to-deploy" [[exemptions.either]] -version = "1.15.0" +version = "1.18.0" criteria = "safe-to-deploy" [[exemptions.equivalent]] @@ -232,7 +220,7 @@ version = "2.5.3" criteria = "safe-to-deploy" [[exemptions.event-listener]] -version = "5.4.1" +version = "5.4.2" criteria = "safe-to-deploy" [[exemptions.event-listener-strategy]] @@ -240,11 +228,11 @@ version = "0.5.4" criteria = "safe-to-deploy" [[exemptions.fastrand]] -version = "2.4.1" +version = "2.5.0" criteria = "safe-to-deploy" [[exemptions.find-msvc-tools]] -version = "0.1.9" +version = "0.1.12" criteria = "safe-to-deploy" [[exemptions.fixedbitset]] @@ -257,7 +245,7 @@ criteria = "safe-to-deploy" [[exemptions.foldhash]] version = "0.1.5" -criteria = "safe-to-deploy" +criteria = "safe-to-run" [[exemptions.foldhash]] version = "0.2.0" @@ -276,19 +264,19 @@ version = "1.2.2" criteria = "safe-to-deploy" [[exemptions.futures]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-channel]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-core]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-executor]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-intrusive]] @@ -296,7 +284,7 @@ version = "0.5.0" criteria = "safe-to-deploy" [[exemptions.futures-io]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-lite]] @@ -304,27 +292,27 @@ version = "2.6.1" criteria = "safe-to-deploy" [[exemptions.futures-macro]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-sink]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-task]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.futures-timer]] -version = "3.0.3" +version = "3.0.4" criteria = "safe-to-deploy" [[exemptions.futures-util]] -version = "0.3.32" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.generic-array]] -version = "0.14.7" +version = "0.14.9" criteria = "safe-to-deploy" [[exemptions.getrandom]] @@ -332,11 +320,7 @@ version = "0.2.17" criteria = "safe-to-deploy" [[exemptions.getrandom]] -version = "0.3.4" -criteria = "safe-to-deploy" - -[[exemptions.getrandom]] -version = "0.4.2" +version = "0.4.3" criteria = "safe-to-deploy" [[exemptions.gloo-timers]] @@ -344,19 +328,23 @@ version = "0.3.0" criteria = "safe-to-deploy" [[exemptions.hashbrown]] -version = "0.15.5" +version = "0.14.5" criteria = "safe-to-deploy" +[[exemptions.hashbrown]] +version = "0.15.5" +criteria = "safe-to-run" + [[exemptions.hashbrown]] version = "0.16.1" criteria = "safe-to-deploy" [[exemptions.hashbrown]] -version = "0.17.0" +version = "0.17.1" criteria = "safe-to-deploy" [[exemptions.hashlink]] -version = "0.11.0" +version = "0.11.1" criteria = "safe-to-deploy" [[exemptions.heck]] @@ -364,7 +352,7 @@ version = "0.5.0" criteria = "safe-to-deploy" [[exemptions.hermit-abi]] -version = "0.5.2" +version = "0.5.3" criteria = "safe-to-deploy" [[exemptions.hex]] @@ -380,47 +368,35 @@ version = "0.13.0" criteria = "safe-to-deploy" [[exemptions.hybrid-array]] -version = "0.4.12" -criteria = "safe-to-deploy" - -[[exemptions.iana-time-zone]] -version = "0.1.65" -criteria = "safe-to-deploy" - -[[exemptions.iana-time-zone-haiku]] -version = "0.1.2" +version = "0.4.15" criteria = "safe-to-deploy" [[exemptions.icu_collections]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_locale_core]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_normalizer]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_normalizer_data]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_properties]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_properties_data]] -version = "2.2.0" +version = "2.3.0" criteria = "safe-to-deploy" [[exemptions.icu_provider]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.id-arena]] -version = "2.3.0" +version = "2.3.1" criteria = "safe-to-deploy" [[exemptions.idna]] @@ -432,7 +408,7 @@ version = "1.2.2" criteria = "safe-to-deploy" [[exemptions.indexmap]] -version = "2.14.0" +version = "2.14.2" criteria = "safe-to-deploy" [[exemptions.itoa]] @@ -440,23 +416,23 @@ version = "1.0.18" criteria = "safe-to-deploy" [[exemptions.js-sys]] -version = "0.3.98" +version = "0.3.105" criteria = "safe-to-deploy" [[exemptions.kv-log-macro]] version = "1.0.7" criteria = "safe-to-deploy" -[[exemptions.leb128fmt]] -version = "0.1.0" -criteria = "safe-to-deploy" +[[exemptions.lazy_static]] +version = "1.5.0" +criteria = "safe-to-run" [[exemptions.libc]] -version = "0.2.186" +version = "0.2.189" criteria = "safe-to-deploy" [[exemptions.libsqlite3-sys]] -version = "0.30.1" +version = "0.37.0" criteria = "safe-to-deploy" [[exemptions.linux-raw-sys]] @@ -464,7 +440,7 @@ version = "0.12.1" criteria = "safe-to-deploy" [[exemptions.litemap]] -version = "0.8.2" +version = "0.8.3" criteria = "safe-to-deploy" [[exemptions.lock_api]] @@ -472,28 +448,32 @@ version = "0.4.14" criteria = "safe-to-deploy" [[exemptions.log]] -version = "0.4.29" +version = "0.4.34" criteria = "safe-to-deploy" +[[exemptions.matchers]] +version = "0.2.0" +criteria = "safe-to-run" + [[exemptions.md-5]] version = "0.11.0" criteria = "safe-to-deploy" [[exemptions.memchr]] -version = "2.8.0" +version = "2.8.3" criteria = "safe-to-deploy" [[exemptions.mio]] -version = "1.2.0" +version = "1.2.3" criteria = "safe-to-deploy" [[exemptions.native-tls]] version = "0.2.18" criteria = "safe-to-deploy" -[[exemptions.num-conv]] -version = "0.2.1" -criteria = "safe-to-deploy" +[[exemptions.nu-ansi-term]] +version = "0.50.3" +criteria = "safe-to-run" [[exemptions.num-traits]] version = "0.2.19" @@ -504,7 +484,7 @@ version = "1.21.4" criteria = "safe-to-deploy" [[exemptions.openssl]] -version = "0.10.79" +version = "0.10.81" criteria = "safe-to-deploy" [[exemptions.openssl-macros]] @@ -516,7 +496,7 @@ version = "0.2.1" criteria = "safe-to-deploy" [[exemptions.openssl-sys]] -version = "0.9.115" +version = "0.9.117" criteria = "safe-to-deploy" [[exemptions.parking]] @@ -540,11 +520,11 @@ version = "0.8.3" criteria = "safe-to-run" [[exemptions.pin-project]] -version = "1.1.12" +version = "1.1.13" criteria = "safe-to-deploy" [[exemptions.pin-project-internal]] -version = "1.1.12" +version = "1.1.13" criteria = "safe-to-deploy" [[exemptions.pin-project-lite]] @@ -560,7 +540,7 @@ version = "0.2.5" criteria = "safe-to-deploy" [[exemptions.pkg-config]] -version = "0.3.33" +version = "0.3.34" criteria = "safe-to-deploy" [[exemptions.polling]] @@ -568,31 +548,15 @@ version = "3.11.0" criteria = "safe-to-deploy" [[exemptions.potential_utf]] -version = "0.1.5" -criteria = "safe-to-deploy" - -[[exemptions.powerfmt]] -version = "0.2.0" -criteria = "safe-to-deploy" - -[[exemptions.ppv-lite86]] -version = "0.2.21" -criteria = "safe-to-deploy" - -[[exemptions.prettyplease]] -version = "0.2.37" +version = "0.1.6" criteria = "safe-to-deploy" [[exemptions.proc-macro2]] -version = "1.0.106" +version = "1.0.107" criteria = "safe-to-deploy" [[exemptions.quote]] -version = "1.0.45" -criteria = "safe-to-deploy" - -[[exemptions.r-efi]] -version = "5.3.0" +version = "1.0.47" criteria = "safe-to-deploy" [[exemptions.r-efi]] @@ -600,19 +564,7 @@ version = "6.0.0" criteria = "safe-to-deploy" [[exemptions.rand]] -version = "0.9.4" -criteria = "safe-to-deploy" - -[[exemptions.rand]] -version = "0.10.1" -criteria = "safe-to-deploy" - -[[exemptions.rand_chacha]] -version = "0.9.0" -criteria = "safe-to-deploy" - -[[exemptions.rand_core]] -version = "0.9.5" +version = "0.10.2" criteria = "safe-to-deploy" [[exemptions.rand_core]] @@ -623,6 +575,14 @@ criteria = "safe-to-deploy" version = "0.5.18" criteria = "safe-to-deploy" +[[exemptions.regex-automata]] +version = "0.4.18" +criteria = "safe-to-run" + +[[exemptions.regex-syntax]] +version = "0.8.11" +criteria = "safe-to-run" + [[exemptions.ring]] version = "0.17.14" criteria = "safe-to-deploy" @@ -632,19 +592,19 @@ version = "1.1.4" criteria = "safe-to-deploy" [[exemptions.rustls]] -version = "0.23.40" +version = "0.23.45" criteria = "safe-to-deploy" [[exemptions.rustls-pki-types]] -version = "1.14.1" +version = "1.15.1" criteria = "safe-to-deploy" [[exemptions.rustls-webpki]] -version = "0.103.13" +version = "0.103.15" criteria = "safe-to-deploy" [[exemptions.rustversion]] -version = "1.0.22" +version = "1.0.23" criteria = "safe-to-deploy" [[exemptions.schannel]] @@ -663,24 +623,20 @@ criteria = "safe-to-deploy" version = "2.17.0" criteria = "safe-to-deploy" -[[exemptions.semver]] -version = "1.0.28" -criteria = "safe-to-deploy" - [[exemptions.serde]] -version = "1.0.228" +version = "1.0.229" criteria = "safe-to-deploy" [[exemptions.serde_core]] -version = "1.0.228" +version = "1.0.229" criteria = "safe-to-deploy" [[exemptions.serde_derive]] -version = "1.0.228" +version = "1.0.229" criteria = "safe-to-deploy" [[exemptions.serde_json]] -version = "1.0.149" +version = "1.0.151" criteria = "safe-to-deploy" [[exemptions.serde_spanned]] @@ -691,6 +647,10 @@ criteria = "safe-to-deploy" version = "0.11.0" criteria = "safe-to-deploy" +[[exemptions.sha1_smol]] +version = "1.0.1" +criteria = "safe-to-deploy" + [[exemptions.sha2]] version = "0.10.9" criteria = "safe-to-deploy" @@ -699,8 +659,12 @@ criteria = "safe-to-deploy" version = "0.11.0" criteria = "safe-to-deploy" +[[exemptions.sharded-slab]] +version = "0.1.7" +criteria = "safe-to-run" + [[exemptions.shlex]] -version = "1.3.0" +version = "2.0.1" criteria = "safe-to-deploy" [[exemptions.slab]] @@ -708,15 +672,15 @@ version = "0.4.12" criteria = "safe-to-deploy" [[exemptions.smallvec]] -version = "1.15.1" +version = "1.16.1" criteria = "safe-to-deploy" [[exemptions.socket2]] -version = "0.6.3" +version = "0.6.5" criteria = "safe-to-deploy" [[exemptions.spin]] -version = "0.9.8" +version = "0.9.9" criteria = "safe-to-deploy" [[exemptions.sqlx]] @@ -760,7 +724,11 @@ version = "2.6.1" criteria = "safe-to-deploy" [[exemptions.syn]] -version = "2.0.117" +version = "2.0.119" +criteria = "safe-to-deploy" + +[[exemptions.syn]] +version = "3.0.5" criteria = "safe-to-deploy" [[exemptions.sync_wrapper]] @@ -768,7 +736,7 @@ version = "1.0.2" criteria = "safe-to-run" [[exemptions.synstructure]] -version = "0.13.2" +version = "0.14.0" criteria = "safe-to-deploy" [[exemptions.tempfile]] @@ -776,51 +744,39 @@ version = "3.27.0" criteria = "safe-to-deploy" [[exemptions.thiserror]] -version = "2.0.18" +version = "2.0.20" criteria = "safe-to-deploy" [[exemptions.thiserror-impl]] -version = "2.0.18" +version = "2.0.20" criteria = "safe-to-deploy" -[[exemptions.time]] -version = "0.3.47" -criteria = "safe-to-deploy" - -[[exemptions.time-core]] -version = "0.1.8" -criteria = "safe-to-deploy" - -[[exemptions.time-macros]] -version = "0.2.27" -criteria = "safe-to-deploy" +[[exemptions.thread_local]] +version = "1.1.10" +criteria = "safe-to-run" [[exemptions.tinystr]] -version = "0.8.3" +version = "0.8.4" criteria = "safe-to-deploy" [[exemptions.tinyvec]] -version = "1.11.0" -criteria = "safe-to-deploy" - -[[exemptions.tinyvec_macros]] -version = "0.1.1" +version = "1.13.3" criteria = "safe-to-deploy" [[exemptions.tokio]] -version = "1.52.2" +version = "1.53.1" criteria = "safe-to-deploy" [[exemptions.tokio-macros]] -version = "2.7.0" +version = "2.7.2" criteria = "safe-to-deploy" [[exemptions.tokio-stream]] -version = "0.1.18" +version = "0.1.19" criteria = "safe-to-deploy" [[exemptions.tokio-util]] -version = "0.7.18" +version = "0.7.19" criteria = "safe-to-run" [[exemptions.toml]] @@ -863,12 +819,20 @@ criteria = "safe-to-deploy" version = "0.1.36" criteria = "safe-to-deploy" +[[exemptions.tracing-log]] +version = "0.2.0" +criteria = "safe-to-run" + +[[exemptions.tracing-subscriber]] +version = "0.3.23" +criteria = "safe-to-run" + [[exemptions.typenum]] -version = "1.20.0" +version = "1.20.1" criteria = "safe-to-deploy" [[exemptions.ulid]] -version = "1.2.1" +version = "3.0.0" criteria = "safe-to-deploy" [[exemptions.unicode-bidi]] @@ -876,7 +840,7 @@ version = "0.3.18" criteria = "safe-to-deploy" [[exemptions.unicode-ident]] -version = "1.0.24" +version = "1.0.25" criteria = "safe-to-deploy" [[exemptions.unicode-normalization]] @@ -887,10 +851,6 @@ criteria = "safe-to-deploy" version = "0.1.4" criteria = "safe-to-deploy" -[[exemptions.unicode-xid]] -version = "0.2.6" -criteria = "safe-to-deploy" - [[exemptions.untrusted]] version = "0.9.0" criteria = "safe-to-deploy" @@ -903,8 +863,16 @@ criteria = "safe-to-deploy" version = "1.0.4" criteria = "safe-to-deploy" +[[exemptions.uuid]] +version = "1.26.1" +criteria = "safe-to-deploy" + +[[exemptions.valuable]] +version = "0.1.1" +criteria = "safe-to-deploy" + [[exemptions.value-bag]] -version = "1.12.0" +version = "1.14.1" criteria = "safe-to-deploy" [[exemptions.vcpkg]] @@ -919,44 +887,24 @@ criteria = "safe-to-deploy" version = "0.11.1+wasi-snapshot-preview1" criteria = "safe-to-deploy" -[[exemptions.wasip2]] -version = "1.0.3+wasi-0.2.9" -criteria = "safe-to-deploy" - -[[exemptions.wasip3]] -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -criteria = "safe-to-deploy" - [[exemptions.wasm-bindgen]] -version = "0.2.121" +version = "0.2.128" criteria = "safe-to-deploy" [[exemptions.wasm-bindgen-futures]] -version = "0.4.71" +version = "0.4.78" criteria = "safe-to-deploy" [[exemptions.wasm-bindgen-macro]] -version = "0.2.121" +version = "0.2.128" criteria = "safe-to-deploy" [[exemptions.wasm-bindgen-macro-support]] -version = "0.2.121" +version = "0.2.128" criteria = "safe-to-deploy" [[exemptions.wasm-bindgen-shared]] -version = "0.2.121" -criteria = "safe-to-deploy" - -[[exemptions.wasm-encoder]] -version = "0.244.0" -criteria = "safe-to-deploy" - -[[exemptions.wasm-metadata]] -version = "0.244.0" -criteria = "safe-to-deploy" - -[[exemptions.wasmparser]] -version = "0.244.0" +version = "0.2.128" criteria = "safe-to-deploy" [[exemptions.web-time]] @@ -964,37 +912,17 @@ version = "1.1.0" criteria = "safe-to-deploy" [[exemptions.webpki-roots]] -version = "1.0.7" +version = "1.0.9" criteria = "safe-to-deploy" [[exemptions.whoami]] -version = "2.1.2" -criteria = "safe-to-deploy" - -[[exemptions.windows-core]] -version = "0.62.2" -criteria = "safe-to-deploy" - -[[exemptions.windows-implement]] -version = "0.60.2" -criteria = "safe-to-deploy" - -[[exemptions.windows-interface]] -version = "0.59.3" +version = "2.1.3" criteria = "safe-to-deploy" [[exemptions.windows-link]] version = "0.2.1" criteria = "safe-to-deploy" -[[exemptions.windows-result]] -version = "0.4.1" -criteria = "safe-to-deploy" - -[[exemptions.windows-strings]] -version = "0.5.1" -criteria = "safe-to-deploy" - [[exemptions.windows-sys]] version = "0.52.0" criteria = "safe-to-deploy" @@ -1043,78 +971,42 @@ criteria = "safe-to-deploy" version = "0.7.15" criteria = "safe-to-deploy" -[[exemptions.wit-bindgen]] -version = "0.51.0" -criteria = "safe-to-deploy" - -[[exemptions.wit-bindgen]] -version = "0.57.1" -criteria = "safe-to-deploy" - -[[exemptions.wit-bindgen-core]] -version = "0.51.0" -criteria = "safe-to-deploy" - -[[exemptions.wit-bindgen-rust]] -version = "0.51.0" -criteria = "safe-to-deploy" - -[[exemptions.wit-bindgen-rust-macro]] -version = "0.51.0" -criteria = "safe-to-deploy" - -[[exemptions.wit-component]] -version = "0.244.0" -criteria = "safe-to-deploy" - -[[exemptions.wit-parser]] -version = "0.244.0" -criteria = "safe-to-deploy" - [[exemptions.writeable]] -version = "0.6.3" +version = "0.6.4" criteria = "safe-to-deploy" [[exemptions.yoke]] -version = "0.8.2" +version = "0.8.3" criteria = "safe-to-deploy" [[exemptions.yoke-derive]] -version = "0.8.2" -criteria = "safe-to-deploy" - -[[exemptions.zerocopy]] -version = "0.8.48" -criteria = "safe-to-deploy" - -[[exemptions.zerocopy-derive]] -version = "0.8.48" +version = "0.8.3" criteria = "safe-to-deploy" [[exemptions.zerofrom]] -version = "0.1.7" +version = "0.1.8" criteria = "safe-to-deploy" [[exemptions.zerofrom-derive]] -version = "0.1.7" +version = "0.1.8" criteria = "safe-to-deploy" [[exemptions.zeroize]] -version = "1.8.2" +version = "1.9.0" criteria = "safe-to-deploy" [[exemptions.zerotrie]] -version = "0.2.4" +version = "0.2.5" criteria = "safe-to-deploy" [[exemptions.zerovec]] -version = "0.11.6" +version = "0.11.8" criteria = "safe-to-deploy" [[exemptions.zerovec-derive]] -version = "0.11.3" +version = "0.11.6" criteria = "safe-to-deploy" [[exemptions.zmij]] -version = "1.0.21" +version = "1.0.23" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 6925c47..238891c 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -28,3 +28,7 @@ audited_as = "1.0.0-rc.6" [[unpublished.apalis-postgres]] version = "1.0.0-rc.8" audited_as = "1.0.0-rc.7" + +[[unpublished.apalis-postgres]] +version = "1.0.0-rc.9" +audited_as = "1.0.0-rc.8"