Web Scraping

Scrape a webpage with custom options

curl --request POST \
  --url https://api.geekflare.com/webscraping \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "https://example.com",
  "device": "desktop",
  "blockAds": true,
  "renderJS": true,
  "proxyCountry": "us",
  "format": "html,markdown",
  "fileOutput": false,
  "stealth": false,
  "waitTime": 2.5,
  "extractionMode": "default",
  "aiPrompt": {
    "type": "prompt",
    "query": "What is the return policy?"
  }
}
'

import requests

url = "https://api.geekflare.com/webscraping"

payload = {
    "url": "https://example.com",
    "device": "desktop",
    "blockAds": True,
    "renderJS": True,
    "proxyCountry": "us",
    "format": "html,markdown",
    "fileOutput": False,
    "stealth": False,
    "waitTime": 2.5,
    "extractionMode": "default",
    "aiPrompt": {
        "type": "prompt",
        "query": "What is the return policy?"
    }
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'https://example.com',
    device: 'desktop',
    blockAds: true,
    renderJS: true,
    proxyCountry: 'us',
    format: 'html,markdown',
    fileOutput: false,
    stealth: false,
    waitTime: 2.5,
    extractionMode: 'default',
    aiPrompt: {type: 'prompt', query: 'What is the return policy?'}
  })
};

fetch('https://api.geekflare.com/webscraping', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.geekflare.com/webscraping",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://example.com',
    'device' => 'desktop',
    'blockAds' => true,
    'renderJS' => true,
    'proxyCountry' => 'us',
    'format' => 'html,markdown',
    'fileOutput' => false,
    'stealth' => false,
    'waitTime' => 2.5,
    'extractionMode' => 'default',
    'aiPrompt' => [
        'type' => 'prompt',
        'query' => 'What is the return policy?'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.geekflare.com/webscraping"

	payload := strings.NewReader("{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.geekflare.com/webscraping")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.geekflare.com/webscraping")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "timestamp": 1783943217189,
  "apiStatus": "success",
  "apiCode": 200,
  "meta": {
    "url": "https://example.com",
    "device": "desktop",
    "format": [
      "html-llm"
    ],
    "fileOutput": false,
    "blockAds": true,
    "renderJS": true,
    "stealth": false,
    "waitTime": 2.5,
    "extractionMode": "default",
    "extractionSchema": {
      "name": "Product Schema",
      "fields": [
        {
          "title": "Product Name",
          "value": "Some hardcoded string"
        }
      ],
      "baseSelector": ".product"
    },
    "test": {
      "id": "mxqx9v9y0742lap6altwdteqd28t23nq"
    },
    "proxyCountry": "<string>",
    "aiPromptType": "prompt"
  },
  "data": "https://example.com/9bulgk075ed9m3vhua5vcrp0.html",
  "aiResult": {}
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 400,
  "message": "INVALID_URL",
  "details": "The URL must be a valid HTTP or HTTPS URL."
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 422,
  "message": "UNABLE_TO_CONNECT",
  "details": "The destination server could not be resolved, refused the connection, timed out, or is redirecting indefinitely."
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 500,
  "message": "CRAWL_FAILED",
  "details": "Our crawling service encountered an error while attempting to fetch data from the specified URL."
}

POST

webscraping

Scrape a webpage with custom options

curl --request POST \
  --url https://api.geekflare.com/webscraping \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "url": "https://example.com",
  "device": "desktop",
  "blockAds": true,
  "renderJS": true,
  "proxyCountry": "us",
  "format": "html,markdown",
  "fileOutput": false,
  "stealth": false,
  "waitTime": 2.5,
  "extractionMode": "default",
  "aiPrompt": {
    "type": "prompt",
    "query": "What is the return policy?"
  }
}
'

import requests

url = "https://api.geekflare.com/webscraping"

payload = {
    "url": "https://example.com",
    "device": "desktop",
    "blockAds": True,
    "renderJS": True,
    "proxyCountry": "us",
    "format": "html,markdown",
    "fileOutput": False,
    "stealth": False,
    "waitTime": 2.5,
    "extractionMode": "default",
    "aiPrompt": {
        "type": "prompt",
        "query": "What is the return policy?"
    }
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'https://example.com',
    device: 'desktop',
    blockAds: true,
    renderJS: true,
    proxyCountry: 'us',
    format: 'html,markdown',
    fileOutput: false,
    stealth: false,
    waitTime: 2.5,
    extractionMode: 'default',
    aiPrompt: {type: 'prompt', query: 'What is the return policy?'}
  })
};

fetch('https://api.geekflare.com/webscraping', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.geekflare.com/webscraping",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://example.com',
    'device' => 'desktop',
    'blockAds' => true,
    'renderJS' => true,
    'proxyCountry' => 'us',
    'format' => 'html,markdown',
    'fileOutput' => false,
    'stealth' => false,
    'waitTime' => 2.5,
    'extractionMode' => 'default',
    'aiPrompt' => [
        'type' => 'prompt',
        'query' => 'What is the return policy?'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.geekflare.com/webscraping"

	payload := strings.NewReader("{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.geekflare.com/webscraping")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.geekflare.com/webscraping")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://example.com\",\n  \"device\": \"desktop\",\n  \"blockAds\": true,\n  \"renderJS\": true,\n  \"proxyCountry\": \"us\",\n  \"format\": \"html,markdown\",\n  \"fileOutput\": false,\n  \"stealth\": false,\n  \"waitTime\": 2.5,\n  \"extractionMode\": \"default\",\n  \"aiPrompt\": {\n    \"type\": \"prompt\",\n    \"query\": \"What is the return policy?\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "timestamp": 1783943217189,
  "apiStatus": "success",
  "apiCode": 200,
  "meta": {
    "url": "https://example.com",
    "device": "desktop",
    "format": [
      "html-llm"
    ],
    "fileOutput": false,
    "blockAds": true,
    "renderJS": true,
    "stealth": false,
    "waitTime": 2.5,
    "extractionMode": "default",
    "extractionSchema": {
      "name": "Product Schema",
      "fields": [
        {
          "title": "Product Name",
          "value": "Some hardcoded string"
        }
      ],
      "baseSelector": ".product"
    },
    "test": {
      "id": "mxqx9v9y0742lap6altwdteqd28t23nq"
    },
    "proxyCountry": "<string>",
    "aiPromptType": "prompt"
  },
  "data": "https://example.com/9bulgk075ed9m3vhua5vcrp0.html",
  "aiResult": {}
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 400,
  "message": "INVALID_URL",
  "details": "The URL must be a valid HTTP or HTTPS URL."
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 422,
  "message": "UNABLE_TO_CONNECT",
  "details": "The destination server could not be resolved, refused the connection, timed out, or is redirecting indefinitely."
}

{
  "timestamp": 1700000000000,
  "apiStatus": "failure",
  "apiCode": 500,
  "message": "CRAWL_FAILED",
  "details": "Our crawling service encountered an error while attempting to fetch data from the specified URL."
}

Authorizations

x-api-key

string

header

required

API Key required for all endpoints

Body

application/json

url

string

required

Target URL

Example:

"https://example.com"

device

enum<string>

default:desktop

Device type to emulate. Defaults to desktop.

Available options:

desktop,

mobile

Example:

"desktop"

blockAds

boolean

default:true

Whether to block ads

Example:

true

renderJS

boolean

default:true

Whether to render JavaScript

Example:

true

proxyCountry

string

Proxy country code to route the request

Example:

"us"

format

enum<string>[]

Format(s) of the scraped result. Comma-separated or array. Defaults to html.

Available options:

html,

markdown,

json,

markdown-llm,

html-llm,

text,

text-llm

Example:

"html,markdown"

fileOutput

boolean

default:false

Whether to get response in file format

Example:

false

stealth

boolean

default:false

Enable stealth mode to bypass basic bot detection (removes webdriver signals, patches navigator properties)

Example:

false

waitTime

number

default:0

Seconds to wait after page load before capturing content. Helps bypass lazy-loaded content and bot checks.

Example:

2.5

extractionMode

enum<string>

default:default

Extraction mode (only used if format=json)

Available options:

default,

cssSchema,

xpathSchema

Example:

"default"

extractionSchema

object

Extraction schema (optional in default mode, required in css/xpath)

Show child attributes

aiPrompt

object

Ask AI to extract or analyze the scraped page. Always runs against the Markdown of the page regardless of the format field. Adds +6 credits on top of the base scraping cost.

Show child attributes

Response

Successfully scraped webpage

timestamp

number

required

Timestamp of the request in milliseconds

Example:

1783943217189

apiStatus

enum<string>

required

API status message

Available options:

success,

failure

Example:

"success"

apiCode

number

required

API status code

Example:

200

Getting started

Endpoint

Authorizations

Body

Response