diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json new file mode 100644 index 0000000..39e0524 --- /dev/null +++ b/.github/plugin/marketplace.json @@ -0,0 +1,19 @@ +{ + "name": "mrscraper", + "owner": { + "name": "MrScraper", + "email": "support@mrscraper.com" + }, + "metadata": { + "description": "MrScraper plugin for GitHub Copilot CLI.", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "mrscraper", + "description": "Retrieve public web pages as HTML, Markdown, or screenshots and discover pages through Google search with MrScraper's hosted MCP server. Includes skills that keep the raw page as the source for local extraction and check account usage.", + "version": "0.1.0", + "source": "./plugins/mrscraper" + } + ] +} diff --git a/.github/scripts/check-plugin.mjs b/.github/scripts/check-plugin.mjs new file mode 100644 index 0000000..11692ee --- /dev/null +++ b/.github/scripts/check-plugin.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +// Repository checks for the MrScraper Copilot CLI plugin. No dependencies. +// Usage: node .github/scripts/check-plugin.mjs [repository root] + +import fs from "node:fs"; +import path from "node:path"; + +const root = path.resolve(process.argv[2] ?? "."); +const pluginDir = path.join(root, "plugins", "mrscraper"); +const errors = []; + +function readJson(relativePath) { + const file = path.join(root, relativePath); + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + errors.push(`${relativePath}: ${error.message}`); + return {}; + } +} + +function check(condition, message) { + if (!condition) { + errors.push(message); + } +} + +const manifest = readJson("plugins/mrscraper/plugin.json"); +const mcp = readJson("plugins/mrscraper/mcp.json"); +const marketplace = readJson(".github/plugin/marketplace.json"); + +// Agent Plugins 1.0 manifest: only plugin.json at the plugin root, and the +// non-empty version and description that the awesome-copilot intake requires. +check(manifest.name === "mrscraper", 'plugin.json: name must be "mrscraper"'); +for (const field of ["version", "description"]) { + check(typeof manifest[field] === "string" && manifest[field].trim() !== "", `plugin.json: ${field} must be a non-empty string`); +} +check(/^\d+\.\d+\.\d+$/.test(manifest.version ?? ""), "plugin.json: version must be MAJOR.MINOR.PATCH"); +check((manifest.description ?? "").length <= 500, "plugin.json: description must be 500 characters or fewer"); +check(Array.isArray(manifest.keywords) && manifest.keywords.length > 0 && manifest.keywords.length <= 10, "plugin.json: keywords must have 1-10 entries"); +for (const keyword of manifest.keywords ?? []) { + check(/^[a-z0-9-]{1,30}$/.test(keyword), `plugin.json: keyword "${keyword}" must be lowercase letters, digits, and hyphens`); +} +for (const legacy of [".plugin/plugin.json", ".claude-plugin/plugin.json", ".github/plugin/plugin.json", ".mcp.json"]) { + check(!fs.existsSync(path.join(pluginDir, legacy)), `plugins/mrscraper/${legacy}: remove it; Agent Plugins 1.0 uses plugin.json and mcp.json at the plugin root`); +} + +// Hosted MCP server: OAuth only, so the configuration must not carry headers. +const servers = Object.entries(mcp.mcpServers ?? {}); +check(servers.length === 1, "mcp.json: expected exactly one MCP server"); +const server = mcp.mcpServers?.mrscraper ?? {}; +check(server.type === "streamable-http", 'mcp.json: mrscraper.type must be "streamable-http"'); +check(server.url === "https://mcp.mrscraper.com/mcp", "mcp.json: mrscraper.url must be https://mcp.mrscraper.com/mcp"); +check(server.headers === undefined, "mcp.json: mrscraper must not define headers; authentication uses OAuth"); + +// Marketplace entry must describe the same plugin and version. +const entries = marketplace.plugins ?? []; +check(entries.length === 1, "marketplace.json: expected exactly one plugin entry"); +const entry = entries[0] ?? {}; +check(entry.name === manifest.name, "marketplace.json: plugin name must match plugin.json"); +check(entry.version === manifest.version, "marketplace.json: plugin version must match plugin.json"); +check(entry.description === manifest.description, "marketplace.json: plugin description must match plugin.json"); +check(marketplace.metadata?.version === manifest.version, "marketplace.json: metadata.version must match plugin.json"); +check(path.resolve(root, entry.source ?? "") === pluginDir, "marketplace.json: source must be ./plugins/mrscraper"); + +// Skills: each directory under skills/ has a SKILL.md whose name matches it. +const expectedSkills = ["mrscraper", "mrscraper-fetch", "mrscraper-serp"]; +const skillsDir = path.join(pluginDir, "skills"); +const foundSkills = fs.existsSync(skillsDir) + ? fs.readdirSync(skillsDir, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name).sort() + : []; +check(JSON.stringify(foundSkills) === JSON.stringify(expectedSkills), `skills: expected ${expectedSkills.join(", ")}; found ${foundSkills.join(", ") || "none"}`); +for (const skill of foundSkills) { + const file = path.join(skillsDir, skill, "SKILL.md"); + if (!fs.existsSync(file)) { + errors.push(`skills/${skill}: SKILL.md is missing`); + continue; + } + const text = fs.readFileSync(file, "utf8"); + const frontmatter = text.match(/^---\n([\s\S]*?)\n---\n/); + check(Boolean(frontmatter), `skills/${skill}/SKILL.md: missing YAML frontmatter`); + const name = frontmatter?.[1].match(/^name:\s*(.+)$/m)?.[1].trim(); + check(name === skill, `skills/${skill}/SKILL.md: frontmatter name "${name}" must equal the directory name`); + check(/^description:\s*\S/m.test(frontmatter?.[1] ?? ""), `skills/${skill}/SKILL.md: frontmatter description is required`); + check(!/\]\(\.\.\//.test(text), `skills/${skill}/SKILL.md: links outside the skill directory fail vally lint`); +} + +// Release notes for the manifest version. +const changelog = fs.readFileSync(path.join(root, "CHANGELOG.md"), "utf8"); +check(changelog.includes(`## ${manifest.version} `), `CHANGELOG.md: add a "## ${manifest.version}" entry`); + +if (errors.length > 0) { + for (const error of errors) { + console.error(`::error::${error}`); + } + process.exit(1); +} + +console.log(`ok: mrscraper ${manifest.version}, ${foundSkills.length} skills, MCP ${server.url}`); diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..7620ce5 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,112 @@ +name: Validate Copilot CLI plugin + +on: + pull_request: + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +env: + COPILOT_CLI_VERSION: 1.0.98 + # github/awesome-copilot commit whose external plugin quality gates this job runs. + AWESOME_COPILOT_SHA: cb64ad12e32737ae764becd6885462e9b4c7cb00 + COPILOT_AUTO_UPDATE: "false" + +jobs: + validate: + name: Schema, repository checks, and install + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Validate against the Agent Plugins 1.0 schemas + run: | + mkdir -p "$RUNNER_TEMP/schemas" + for name in plugin mcp; do + curl -fsSL -o "$RUNNER_TEMP/schemas/$name.schema.json" \ + "https://agent-plugins.org/schemas/1.0.0/$name.schema.json" + npx --yes ajv-cli@5.0.0 validate --spec=draft2020 \ + -s "$RUNNER_TEMP/schemas/$name.schema.json" \ + -d "plugins/mrscraper/$name.json" + done + + - name: Check manifest, marketplace, skills, and changelog + run: node .github/scripts/check-plugin.mjs . + + - name: Install Copilot CLI + run: npm install --global "@github/copilot@${COPILOT_CLI_VERSION}" + + - name: Install the plugin and confirm what it loads + env: + HOME: ${{ runner.temp }}/copilot-home + run: | + mkdir -p "$HOME" + copilot --version + copilot plugin marketplace add "$GITHUB_WORKSPACE" + copilot plugin install mrscraper@mrscraper + + copilot plugin list --json | tee "$RUNNER_TEMP/plugins.json" + version="$(jq -r .version plugins/mrscraper/plugin.json)" + jq -e --arg version "$version" \ + 'any(.[]; .name == "mrscraper" and .marketplace == "mrscraper" and .version == $version and .enabled)' \ + "$RUNNER_TEMP/plugins.json" + + copilot mcp get mrscraper --json | tee "$RUNNER_TEMP/mcp.json" + jq -e '.mrscraper | .url == "https://mcp.mrscraper.com/mcp" and .source == "plugin" and (has("headers") | not)' \ + "$RUNNER_TEMP/mcp.json" + + copilot skill list | tee "$RUNNER_TEMP/skills.txt" + for skill in mrscraper mrscraper-fetch mrscraper-serp; do + grep -qE "^ ${skill} - " "$RUNNER_TEMP/skills.txt" || { echo "::error::missing skill ${skill}"; exit 1; } + done + + awesome-copilot-gates: + name: awesome-copilot external plugin quality gates + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Check out awesome-copilot quality gates + uses: actions/checkout@v4 + with: + repository: github/awesome-copilot + ref: ${{ env.AWESOME_COPILOT_SHA }} + path: awesome-copilot + sparse-checkout: eng + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install gate dependencies and Copilot CLI + run: | + npm ci --prefix awesome-copilot --ignore-scripts --no-audit --no-fund + npm install --global "@github/copilot@${COPILOT_CLI_VERSION}" + + - name: Run spec compliance, vally lint, install smoke test, and version match + env: + SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + plugin_json="$(jq -c --arg repo "$GITHUB_REPOSITORY" --arg sha "$SOURCE_SHA" \ + '{name, description, version, author, repository, homepage, license, keywords, + source: {source: "github", repo: $repo, path: "plugins/mrscraper", sha: $sha}}' \ + plugins/mrscraper/plugin.json)" + cd awesome-copilot + node eng/external-plugin-quality-gates.mjs --plugin-json "$plugin_json" \ + --log-file "$RUNNER_TEMP/quality-gates.log" > "$RUNNER_TEMP/quality-gates.json" + cat "$RUNNER_TEMP/quality-gates.log" + jq -e '.overall_status == "pass" and .spec_compliance_status == "pass"' "$RUNNER_TEMP/quality-gates.json" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..255b11b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.mrscraper/ +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1232879 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## 0.1.0 - 2026-10-16 + +- First release of the MrScraper plugin for GitHub Copilot CLI, in the Agent + Plugins 1.0 format. +- Hosted MCP server `https://mcp.mrscraper.com/mcp` (Streamable HTTP, OAuth 2.1 + with dynamic client registration) exposing `scraperAPI`, `webSearch`, and + `status`. +- Skills: `mrscraper`, `mrscraper-fetch`, and `mrscraper-serp`. +- Marketplace file at `.github/plugin/marketplace.json` for + `copilot plugin install mrscraper@mrscraper`. +- CI that validates the manifests, installs the plugin with Copilot CLI, and + runs the awesome-copilot external plugin quality gates. diff --git a/README.md b/README.md new file mode 100644 index 0000000..213d509 --- /dev/null +++ b/README.md @@ -0,0 +1,197 @@ +# MrScraper for GitHub Copilot CLI + +MrScraper connects [GitHub Copilot CLI](https://docs.github.com/copilot/how-tos/copilot-cli) +to a hosted MCP server that retrieves public web pages as HTML, Markdown, or +screenshots, discovers pages through Google search, and reports account usage. +The plugin also includes three skills that tell the agent which tool to use and +to keep the raw page content as the source for summaries and local extraction. + +## What this plugin adds + +- The `mrscraper` MCP server: Streamable HTTP at `https://mcp.mrscraper.com/mcp`. +- OAuth 2.1 sign-in through Copilot CLI's MCP client, with dynamic client + registration. This repository stores no credentials. +- Skills: `mrscraper`, `mrscraper-fetch`, and `mrscraper-serp`. + +The plugin uses the [Agent Plugins 1.0](https://agent-plugins.org/specification) +layout: `plugin.json`, `mcp.json`, and `skills/` in `plugins/mrscraper`. + +## When to use it + +Copilot CLI's built-in `web_fetch` tool and web search are enough for many +public pages. This plugin helps when a task needs more control over retrieval: + +- Pages that render their content with JavaScript, or that need a CSS selector + to appear before the content is read. +- Content that differs by country, with retries and a per-request token cap. +- A screenshot of the page saved as an image file. +- Google results as structured JSON (organic results, related questions, and + pagination) for a chosen region, language, and results page. + +It retrieves pages; it does not drive a browser session. For clicking, form +filling, or signed-in flows, use Playwright or another browser automation tool. + +## Requirements + +- GitHub Copilot CLI. Installation was tested with 1.0.98. +- A [MrScraper](https://app.mrscraper.com) account. The free plan includes + 1,000 tokens per month and does not require a credit card. +- A local browser for the OAuth sign-in. +- Permission to access and process the target content. + +## Install + +```bash +copilot plugin marketplace add mrscraper-com/mrscraper-copilot-plugin +copilot plugin install mrscraper@mrscraper +``` + +Confirm what the plugin loaded: + +```bash +copilot plugin list +copilot mcp list # Plugin servers: mrscraper (http) +copilot skill list # mrscraper, mrscraper-fetch, mrscraper-serp +``` + +In an interactive session, `/plugin list`, `/mcp list`, and `/skills list` show +the same information. + +## Sign in + +Start `copilot` and run: + +```text +/mcp auth mrscraper +``` + +Copilot CLI discovers MrScraper's authorization server, registers itself, and +opens your browser. Approve access on the MrScraper consent page; Copilot CLI +stores and refreshes the tokens. Until you sign in, Copilot CLI reports that +the `mrscraper` server requires authentication. Never paste OAuth tokens or API +keys into the chat. + +`/mcp show mrscraper` then lists these tools: + +| Tool | Purpose | +| --- | --- | +| `scraperAPI` | Retrieve a known page as HTML, Markdown, a screenshot, or a combination. | +| `webSearch` | Discover public pages through Google. | +| `status` | Inspect subscription status, token usage, and remaining tokens. | + +The OAuth scopes are `scrape:read` (`scraperAPI` and `webSearch`) and +`account:read` (`status`). + +## Try it + +```text +Fetch https://www.scrapethissite.com/pages/simple/ as Markdown and list the five most populous countries. +``` + +```text +Search Google for open source web scraping libraries and compare the top five results. +``` + +```text +How many MrScraper tokens do I have left this month? +``` + +Copilot CLI asks for approval before it runs MCP tools. To approve the +MrScraper tools for one non-interactive run, pass `--allow-tool='mrscraper'`: + +```bash +copilot -p "Fetch https://example.com as Markdown and summarize it." --allow-tool='mrscraper' +``` + +## Retrieve-first routing + +When a public URL is already known, the skills direct the agent to retrieve it +with `scraperAPI` first and treat the raw response as the source of truth. The +agent can then read, summarize, compare, or derive structured output locally +without losing details. When no URL is known, `webSearch` finds candidate pages +first. + +## API key fallback + +Use OAuth whenever possible. If OAuth cannot complete, for example on a machine +without a browser, create an [API key](https://app.mrscraper.com/api-tokens), +export it as `MRSCRAPER_API_KEY`, add a user-level server under a different +name, and disable the plugin's OAuth server so the tools are not listed twice: + +```bash +copilot mcp add --transport http \ + --header "Authorization: Bearer $MRSCRAPER_API_KEY" \ + mrscraper-api-key https://mcp.mrscraper.com/mcp +copilot mcp disable mrscraper +``` + +The skills match tools by name, so they keep working with either server. An +API key carries full account authority, and `copilot mcp add` writes the header +to `~/.copilot/mcp-config.json` in plain text; never add it to a project's +`.mcp.json` or `.github/mcp.json` that is committed. To return to OAuth, run +`copilot mcp remove mrscraper-api-key` and `copilot mcp enable mrscraper`. + +## Data and permissions + +The plugin sends MCP tool inputs, including target URLs and search queries, to +MrScraper's hosted service. Page responses and tool results are then available +to the agent for the requested task. Use it only with public or otherwise +authorized content, and follow the target site's requirements. + +Review MrScraper's [MCP documentation](https://docs.mrscraper.com/docs/getting-started/mcp-server), +[Privacy Policy](https://mrscraper.com/privacy-policy), +[Terms of Use](https://mrscraper.com/terms-of-use), and +[Acceptable Use Policy](https://mrscraper.com/acceptable-use-policy) before use. + +## Update or remove + +```bash +copilot plugin update mrscraper +copilot plugin uninstall mrscraper +copilot plugin marketplace remove mrscraper +``` + +## Support and security + +- Product help: [MrScraper MCP documentation](https://docs.mrscraper.com/docs/getting-started/mcp-server) +- Bugs and feature requests: [GitHub Issues](https://github.com/mrscraper-com/mrscraper-copilot-plugin/issues) +- Account help: [support@mrscraper.com](mailto:support@mrscraper.com) +- Security reports: see [SECURITY.md](SECURITY.md) + +## Development + +```text +.github/plugin/marketplace.json marketplace with one plugin entry +plugins/mrscraper/plugin.json Agent Plugins 1.0 manifest +plugins/mrscraper/mcp.json hosted MCP server (Streamable HTTP, OAuth) +plugins/mrscraper/skills/ mrscraper, mrscraper-fetch, mrscraper-serp +.github/scripts/check-plugin.mjs repository checks used by CI +``` + +Install a local checkout and confirm that it loads: + +```bash +copilot plugin marketplace add ./ +copilot plugin install mrscraper@mrscraper +copilot mcp get mrscraper +copilot skill list +node .github/scripts/check-plugin.mjs . +``` + +A plugin installed from a local marketplace is loaded live from the checkout, +so edits take effect in the next session. CI validates `plugin.json` and +`mcp.json` against the Agent Plugins 1.0 schemas, runs the repository checks, +installs the plugin with Copilot CLI, and runs the +[awesome-copilot](https://github.com/github/awesome-copilot) external plugin +quality gates (spec compliance, `vally lint`, install smoke test, and version +match). + +The skills are copied from the canonical MrScraper MCP skills, with links to +sibling skills written as skill names because `vally lint` only accepts file +references inside a skill's own directory. Keep the version in `plugin.json` +and `.github/plugin/marketplace.json` in step, and record user-visible changes +in [CHANGELOG.md](CHANGELOG.md). + +## License + +[MIT](LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..16dba8a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +Please report suspected vulnerabilities privately to +[support@mrscraper.com](mailto:support@mrscraper.com). Include the affected +component, reproduction steps, and potential impact. Do not open a public issue +for an unpatched vulnerability or include credentials, tokens, customer data, +or private target content in a report. + +This repository contains no MrScraper credentials and no executable code that +runs on your machine: the plugin is a manifest, an MCP server URL, and skill +instructions. Authentication is handled by GitHub Copilot CLI's OAuth 2.1 MCP +connection, which stores and refreshes the tokens. If you use the API key +fallback, the key stays in your own `~/.copilot/mcp-config.json` and carries +full account authority. If a credential may have been exposed, revoke it +through the relevant account or client immediately and then contact support. diff --git a/plugins/mrscraper/mcp.json b/plugins/mrscraper/mcp.json new file mode 100644 index 0000000..33765d0 --- /dev/null +++ b/plugins/mrscraper/mcp.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", + "mcpServers": { + "mrscraper": { + "type": "streamable-http", + "url": "https://mcp.mrscraper.com/mcp" + } + } +} diff --git a/plugins/mrscraper/plugin.json b/plugins/mrscraper/plugin.json new file mode 100644 index 0000000..7213917 --- /dev/null +++ b/plugins/mrscraper/plugin.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "mrscraper", + "version": "0.1.0", + "description": "Retrieve public web pages as HTML, Markdown, or screenshots and discover pages through Google search with MrScraper's hosted MCP server. Includes skills that keep the raw page as the source for local extraction and check account usage.", + "author": { + "name": "MrScraper", + "email": "support@mrscraper.com", + "url": "https://mrscraper.com" + }, + "homepage": "https://docs.mrscraper.com/docs/getting-started/mcp-server", + "repository": "https://github.com/mrscraper-com/mrscraper-copilot-plugin", + "license": "MIT", + "keywords": ["web-scraping", "web-search", "serp", "markdown", "screenshot", "mcp", "mrscraper"] +} diff --git a/plugins/mrscraper/skills/mrscraper-fetch/SKILL.md b/plugins/mrscraper/skills/mrscraper-fetch/SKILL.md new file mode 100644 index 0000000..41aa14a --- /dev/null +++ b/plugins/mrscraper/skills/mrscraper-fetch/SKILL.md @@ -0,0 +1,195 @@ +--- +name: mrscraper-fetch +description: | + Retrieve a known public URL with MrScraper's scraperAPI tool as HTML, Markdown, a screenshot, or a combination. Use for reading, summarizing, archiving, or analyzing a page and before local extraction; supports automatic browser fallback, real-device routing, proxies, and retries. Use mrscraper-serp when no URL is known. +--- + +# Retrieve Page Content with MrScraper MCP + +Use the scraperAPI tool supplied by the MrScraper MCP server whenever the user +already has a public URL. Keep using the retrieved content for summarization, +comparison, transformation, and structured output. Use +`mrscraper` for connection and authentication checks, +account status, or broader routing, and +`mrscraper-serp` when discovery must happen first. + +## Page Retrieval Context + +scraperAPI calls MrScraper's +[page-fetching service](https://docs.mrscraper.com/docs/features/unblocker) once. +Browser rendering executes page JavaScript, geographic routing selects the +proxy country, selector waits let delayed content appear, and homepage +navigation visits the site root before the target. Real-device routing (super) +is independent of browser rendering. + +Use scraperAPI only for content the user is authorized to access and in +accordance with the site's requirements. + +Start with the URL alone and add only the controls the target needs. Plan-token +usage is based on runtime and bandwidth: one token per 30 seconds and one token +per 0.25 MB, rounded up per component. Resource blocking can reduce bandwidth +for text-focused pages. Retries stop when the request succeeds, max_retries is +reached, or the running token total reaches token_cap. The initial request +always runs even when it exceeds that cap. See the +[Token Plan](https://docs.mrscraper.com/docs/getting-started/api-token) for the +maintained calculation. + +## Step 1 — Define the Outcome + +Confirm the target URL and what the user wants: + +- Read, summarize, cite, or inspect the page. +- Check whether specific text appears. +- Archive the page or capture a screenshot. +- Produce fields, JSON, tables, or other structured output with local logic. +- Load JavaScript-rendered or geo-sensitive content. + +When many pages share a layout, retrieve representative pages, define one local +extractor, retrieve the remaining pages in parallel when safe and proportional, +and apply the same extractor to every saved response. + +## Step 2 — Retrieve the Page + +The client may namespace the tool name; select scraperAPI from the MrScraper +MCP provider. Start with one call containing only the required URL, which +requests HTML: + + { + "url": "https://www.scrapethissite.com/pages/simple/" + } + +Request Markdown instead of HTML for readable text: + + { + "url": "https://www.scrapethissite.com/pages/simple/", + "html": false, + "markdown": true + } + +The result is available in MCP structuredContent and as formatted JSON text. +The requested representations are in data: + + { + "status_code": 200, + "data": { + "html": "...", + "markdown": null, + "screenshot": null, + "token_usage": 3, + "runtime": 18.02, + "retry_count": 0, + "error": null + }, + "headers": { + "x-status-code": "200" + } + } + +Prefer structuredContent. Check isError, error, status_code, and +headers["x-status-code"] (the target page's status) before using data. + +A screenshot is a base64 JPEG that can be large. Request it only when the user +needs an image, and save it to a file rather than reading it into the +conversation: + + { + "url": "https://www.scrapethissite.com/pages/simple/", + "html": false, + "screenshot": true + } + +## Step 3 — Choose Page-Loading Options + +Omit browser_rendering to let the service choose: it falls back to a browser +automatically when needed. true forces a browser and false disables it. +screenshot=true always uses a browser. + +Use real-device routing when routing may be the problem: + + { + "url": "https://www.scrapethissite.com/pages/simple/", + "super": true + } + +Wait for delayed content with a CSS selector: + + { + "url": "https://www.scrapethissite.com/pages/ajax-javascript/#2015", + "browser_rendering": true, + "wait_for_selector": ".film" + } + +Use geographic routing or homepage navigation when the target requires it: + + { + "url": "https://www.scrapethissite.com/pages/simple/", + "geo_code": "ID", + "home_page": true + } + +Bound resource use: + + { + "url": "https://www.scrapethissite.com/pages/ajax-javascript/#2015", + "block_resources": true, + "max_retries": 1, + "token_cap": 10, + "timeout": 60 + } + +### Parameters + +| MCP input | Default | Request mapping | Use | +| --- | --- | --- | --- | +| url | required | Body url | Absolute HTTP or HTTPS target URL. | +| html | true | Query html | Request HTML content. | +| markdown | false | Query markdown | Request Markdown content. | +| screenshot | false | Query screenshot | Capture a base64 JPEG; forces browser rendering. | +| browser_rendering | automatic | Query browserRendering | true forces and false disables browser rendering. | +| super | false | Query super | Real-device routing, independent of browser rendering. | +| geo_code | US | Query geoCode | Proxy country, such as US, ID, or SG. | +| proxy_country | omitted | Query proxyCountry | Compatibility country option. | +| use_proxy | true | Query useProxy | Set false to skip the service's configured proxy. | +| proxy | omitted | Query proxy | Custom proxy as username:password@host:port. | +| wait_for_selector | omitted | Query waitForSelector | CSS selector to wait for. | +| home_page | false | Query warmStart | Visit the site root before the target page. | +| block_resources | false | Query blockResources | Block images, CSS, and fonts. | +| bypass_proxy | false | Query bypassProxy | Enable the proxy-bypass behavior. | +| cookie_jar | omitted | Query cookieJar | Cookie configuration string. | +| headers | omitted | Query headers | Target page request headers as an object or JSON string. | +| max_retries | 3 | Body maxRetries | Maximum retries; zero disables retries. | +| token_cap | omitted | Body tokenCap | Running token total that stops further retries. | +| timeout | 120 | Query timeout | Page timeout from 1 to 300 seconds. | +| stream | omitted | Query stream | Ask the API to stream; the response is still returned once. | +| discard_token_secret | empty | Header x-discard-token-secret | Optional upstream header. | + +Authentication is handled by the MCP client's OAuth 2.1 connection and is +never a tool input. + +## Step 4 — Inspect and Retry Deliberately + +Start with the URL alone unless the task already establishes a requirement. If +the response fails, is blocked, incomplete, or missing dynamic content: + +1. Inspect status_code, data.error, headers["x-status-code"], and the content. +2. Change one axis at a time: force or disable browser_rendering, or set super + when routing may be the problem. +3. Add wait_for_selector, geo_code, or home_page only when evidence shows that + the target requires it. +4. Stop after a usable response unless the user requests a comparison. + +Do not repeat an identical request. Treat wait_for_selector as a CSS selector, +not a duration. Browser rendering loads a page; it does not click controls, +submit forms, or provide an authenticated interactive browser session. + +## Step 5 — Deliver the Result + +Answer the user's request from data. Keep the full envelope when headers or +diagnostics matter. When the user requests an archive, save the successful +response or the requested field with the environment's local file-writing +capability; scraperAPI itself has no output-path input. + +Keep retrieved content as the source of truth for later steps. Build +summaries, tables, JSON transformations, and extraction scripts from it, and +retrieve the page again when a follow-up needs details that an earlier +transformation did not preserve. diff --git a/plugins/mrscraper/skills/mrscraper-serp/SKILL.md b/plugins/mrscraper/skills/mrscraper-serp/SKILL.md new file mode 100644 index 0000000..9a49c33 --- /dev/null +++ b/plugins/mrscraper/skills/mrscraper-serp/SKILL.md @@ -0,0 +1,116 @@ +--- +name: mrscraper-serp +description: | + Discover public pages through Google with MrScraper's webSearch tool using a search query or Google search URL. Use when no target URL is known or the user asks to inspect Google results. After discovery, retrieve selected pages with mrscraper-fetch. +--- + +# Discover Pages with MrScraper MCP Web Search + +Use the webSearch tool supplied by the MrScraper MCP server when a task begins +with a search query rather than a known target URL. Use +`mrscraper` for connection troubleshooting, account +status, or broader routing. + +## Step 1 — Build the Query + +Include the important entity, attribute, location, or date in url, and set +raw=false to receive parsed results: + + { + "url": "web scraping best practices", + "raw": false + } + +The input can also be a complete Google search URL: + + { + "url": "https://www.google.com/search?q=web+scraping&gl=us&hl=en&start=20", + "raw": false + } + +For a Google URL, the tool reads q, gl, hl, and start. Explicit region, +language, and page inputs take precedence. + +## Step 2 — Set Locale and Pagination + +Set both country and language when results must match a locale: + + { + "url": "running shoes", + "raw": false, + "region": "id", + "language": "id", + "page": 2 + } + +Start with the first page unless the user requests broader coverage. Inspect +the current page before requesting another. + +## Step 3 — Choose the Output + +raw defaults to true, which returns result-page HTML as the Python SDK does. +Set raw=false for parsed JSON; organic results are in data.data.organic, next +to sections such as people_also_ask and pagination. + +Request result-page HTML when the page itself is needed: + + { + "url": "iphone 17" + } + +Use JavaScript rendering for dynamic result features such as AI Overview: + + { + "url": "what is web scraping", + "raw": false, + "render_js": true + } + +### Parameters + +| MCP input | Default | Request mapping | Use | +| --- | --- | --- | --- | +| url | required | Body query | Search query or complete Google search URL. | +| raw | true | Body format=html | Return result-page HTML; set false to use format. | +| format | json | Body format | json or html when raw=false. | +| region | URL value or omitted | Body region | Google result country, such as us or id. | +| language | URL value or omitted | Body language | Google result language, such as en or id. | +| page | URL value or omitted | Body page | Positive, one-based results page. | +| render_js | false | Body renderJs | Render JavaScript-dependent result features. | +| timeout | 600 | MCP upstream deadline | Maximum wait in seconds; not sent in the search body. | + +Authentication is handled by the MCP client's OAuth 2.1 connection and is +never a tool input. + +## Step 4 — Inspect the Results + +The complete response envelope is available in MCP structuredContent and as +formatted JSON text. Prefer structuredContent. Check isError, error, and +status_code before using data. + +When the user requests a search artifact, save the successful data or full +envelope with local file-writing capability; webSearch has no output-path +input. + +## Step 5 — Continue with Relevant URLs + +Select only URLs relevant to the user's goal, then load +`mrscraper-fetch` to retrieve and inspect each +selected page before deriving answers, comparisons, or structured output. + +Run independent follow-up URLs in parallel when the environment supports safe +parallel execution. Keep the number of pages proportional to the requested +coverage. If the user only requested URL discovery, stop after the search. + +## Step 6 — Handle Weak or Missing Results + +- Set both region and language when the locale is wrong. +- Rewrite noisy queries around the important entity, attribute, and location. +- Request another result page only after checking the current page. +- Retry once with render_js=true when a dynamic result feature is required. +- Increase timeout when the MCP request deadline is too short. +- Use `mrscraper` when the connection or authentication + fails. + +webSearch is for public Google discovery, not for discovering URLs within one +known site. diff --git a/plugins/mrscraper/skills/mrscraper/SKILL.md b/plugins/mrscraper/skills/mrscraper/SKILL.md new file mode 100644 index 0000000..565afd2 --- /dev/null +++ b/plugins/mrscraper/skills/mrscraper/SKILL.md @@ -0,0 +1,172 @@ +--- +name: mrscraper +description: | + Connect, authenticate, route, and troubleshoot MrScraper MCP, and check account usage with the status tool. For known public URLs, retrieve and preserve page content with mrscraper-fetch first; use mrscraper-serp for query-first discovery through Google. +--- + +# MrScraper MCP + +Use this skill for connection and authentication checks, tool selection, shared +response handling, account status, and troubleshooting. Load the focused fetch +or SERP skill for page retrieval and Google search. + +## Step 1 — Verify or Connect MrScraper MCP + +Check whether the current client exposes these three tools from the MrScraper +MCP provider: + + scraperAPI webSearch status + +The client may display server-qualified names. Match the provider and +unqualified tool name; do not assume a fixed namespace. + +If the tools are already available, use the existing connection. If they are +missing, ask the user to enable the existing MrScraper connection or configure +the managed Streamable HTTP endpoint: + + https://mcp.mrscraper.com/mcp + +A self-managed deployment may expose the same tools over Streamable HTTP or +stdio. Connection setup belongs to the MCP client. After setup or reconnection, +list the available tools and confirm all three names before claiming the +integration is ready. + +## Step 2 — Authenticate + +MrScraper MCP authenticates through OAuth 2.1. Authentication belongs to the +MCP connection and is never a tool argument. + +When the connection is not authorized, start the MCP client's authorization +flow. Let the client discover the authorization metadata, open the MrScraper +consent page, complete authorization, store the resulting credentials, and +refresh access tokens. The user may need to approve access in a browser, but +must not paste an authorization code, access token, refresh token, API key, or +other credential into the conversation. + +Never place credentials in tool arguments, print them, write them into project +artifacts, or manually reproduce the OAuth exchange. Do not configure an +API-key or bearer-token fallback from these skills. + +The connection uses two scopes: scrape:read for scraperAPI and webSearch, and +account:read for status. A tool called without its scope returns 403 +insufficient_scope; reauthorize the connection so the client requests it. + +Verify end-to-end access with status, which reads the account without +retrieving a page: + + {} + +A successful account summary confirms that the connection can authenticate and +reach MrScraper. If it fails, report the error and do not claim setup is +complete. + +## Step 3 — Route the Request + +| User outcome | MCP tool or skill | +| --- | --- | +| Retrieve and preserve a known public page as HTML, Markdown, or a screenshot | `mrscraper-fetch` with scraperAPI | +| Discover relevant pages from a Google query | `mrscraper-serp` with webSearch, then retrieve selected pages | +| Check the plan, token usage, or remaining tokens | status | + +### Retrieve first, extract locally + +scraperAPI returns raw page content. Produce summaries, tables, JSON, and other +structured output from that content with local analysis or extraction logic. +The raw response preserves complete page context for later questions, +validation, and changes to the requested schema. + +Prefer this workflow for pages with a shared layout: + +1. Retrieve representative pages and inspect their content. +2. Understand the shared page structure and required fields. +3. Define a stable output schema and reusable local extractor. +4. Retrieve the remaining pages, in parallel when safe. +5. Apply the same extractor to every saved response. + +Request Markdown when readable text is enough and HTML when an extractor needs +the page structure. + +For discovery-first work, call webSearch, select relevant URLs, and retrieve +the pages whose content will inform the answer. + +MrScraper MCP does not create AI scrapers, rerun saved scrapers, or list stored +results. Point the user to the MrScraper dashboard at https://app.mrscraper.com +for those. + +## Step 4 — Handle Output and Artifacts + +scraperAPI and webSearch return the same response envelope in MCP +structuredContent and in a formatted JSON text block: + + { + "status_code": 200, + "data": {}, + "headers": {} + } + +Prefer structuredContent when available. An API failure adds error and sets the +MCP result's isError flag. An input-contract failure is returned as a tool +error. Check those signals before trusting data. Credential-bearing headers and +credential metadata are removed from responses. + +MrScraper tools do not accept output paths. When the user requests an artifact, +wait for a successful tool result, select the requested value, and save it with +the environment's local file-writing capability. A screenshot is a base64 JPEG +in data.screenshot; decode it before saving an image. + +Save substantial intermediate artifacts under the current project's +.mrscraper directory unless the user specifies another location. This project +artifact directory is unrelated to MCP connection credentials. Keep artifacts +out of version control unless the user asks to commit them. + +## Step 5 — Review Account Usage + +Call status with an empty object: + + {} + +The result summarizes the subscription status, token limit, token usage, +remaining tokens, usage percentage, rate limit, auto-renewal, renewal date, +and the account's name, email, and verification state: + + { + "kind": "mrscraper-cli-status-summary", + "status_code": 200, + "data": { + "account": { + "subscription_status": "active", + "token_usage": 250, + "token_limit": 1000, + "token_remaining": 750, + "usage_percent": 25 + } + } + } + +Check remaining tokens before a large batch of page retrievals. status reports +account usage, not the progress of a request. + +## Step 6 — Troubleshoot + +- Missing tools: enable or reconnect the MrScraper MCP connection, then confirm + scraperAPI, webSearch, and status are exposed. +- Unauthorized or expired authorization: reauthorize the OAuth 2.1 connection + through the MCP client. +- 403 insufficient_scope: reauthorize so the client requests the missing scope. +- Input-contract error: inspect the tool schema and omit inputs that do not + apply. The previous fetch, scrape, serp, rerun, results, and result tools were + replaced by scraperAPI, webSearch, and status. +- Page incomplete or missing dynamic content: load mrscraper-fetch and revise + page-loading inputs deliberately. +- Search results weak: load mrscraper-serp and refine the query, locale, or + page. + +## Limits + +Use another tool for: + +- Clicking controls, completing forms, or authenticated browser sessions. +- Parsing local PDFs, documents, spreadsheets, or other files. +- Recurring monitoring, notifications, or scheduling. + +Explain the boundary and continue with any supported portion of the task.