JSON DOM Response
Get your web scraping results in JSON format.
Traditional web scraping usually results in extracting the raw HTML from a website and using it with an HTML parser in order to manipulate the data. However, certain scenarios may require a more structured result. In these scenarios, returning the data in a JSON format may be the best option.
With WebScrapingAPI, you can enable the json_dom option, which will return the scraped website in a JSON format. Moreover, this option can be used both when render_js is turned on or off.
Your full GET request should then be sent to the following address:
https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=<TARGETED_URL>&json_dom=1JSON DOM Response Integration Examples
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://httpbin.org/&json_dom=1"const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"json_dom":1
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response.text)<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1")
.asString();var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org&json_dom=1")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_bodyImportant! The url parameter has to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F )
Response Example
{
"0":{
"tagName":"a",
"attributes":{
"href":"https://github.com/requests/httpbin",
"class":"github-corner",
"aria-label":"View source on Github"
},
"children":{
"0":{
"tagName":"svg",
"attributes":{
"width":"80",
"height":"80",
"viewBox":"0 0 250 250",
"style":"fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;",
"aria-hidden":"true"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"
}
},
"1":{
"tagName":"path",
"attributes":{
"d":"M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2",
"fill":"currentColor",
"style":"transform-origin: 130px 106px;",
"class":"octo-arm"
}
},
"2":{
"tagName":"path",
"attributes":{
"d":"M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z",
"fill":"currentColor",
"class":"octo-body"
}
}
}
}
}
},
"1":{
"tagName":"svg",
"attributes":{
"xmlns":"http://www.w3.org/2000/svg",
"xlink":"http://www.w3.org/1999/xlink",
"style":"position:absolute;width:0;height:0"
},
"children":{
"0":{
"tagName":"defs",
"children":{
"0":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 20 20",
"id":"unlocked"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"
}
}
}
},
"1":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 20 20",
"id":"locked"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"
}
}
}
},
"2":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 20 20",
"id":"close"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"
}
}
}
},
"3":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 20 20",
"id":"large-arrow"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"
}
}
}
},
"4":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 20 20",
"id":"large-arrow-down"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"
}
}
}
},
"5":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 24 24",
"id":"jump-to"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"
}
}
}
},
"6":{
"tagName":"symbol",
"attributes":{
"viewBox":"0 0 24 24",
"id":"expand"
},
"children":{
"0":{
"tagName":"path",
"attributes":{
"d":"M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"
}
}
}
}
}
}
}
},
"2":{
"tagName":"div",
"attributes":{
"id":"swagger-ui"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"data-reactroot":"",
"class":"swagger-ui"
},
"children":{
"0":{
"tagName":"div",
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"information-container wrapper"
},
"children":{
"0":{
"tagName":"section",
"attributes":{
"class":"block col-12"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"info"
},
"children":{
"0":{
"tagName":"hgroup",
"attributes":{
"class":"main"
},
"children":{
"0":{
"tagName":"h2",
"attributes":{
"class":"title"
},
"text":"httpbin.org",
"children":{
"0":{
"tagName":"small",
"children":{
"0":{
"tagName":"pre",
"attributes":{
"class":"version"
},
"text":"0.9.2"
}
}
}
}
},
"1":{
"tagName":"pre",
"attributes":{
"class":"base-url"
},
"text":"[ Base URL: httpbin.org/ ]"
}
}
},
"1":{
"tagName":"div",
"attributes":{
"class":"description"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"markdown"
},
"children":{
"0":{
"tagName":"p",
"text":"A simple HTTP Request & Response Service.",
"children":{
"0":{
"tagName":"br"
},
"1":{
"tagName":"br"
},
"2":{
"tagName":"b",
"text":"Run locally:"
},
"3":{
"tagName":"code",
"text":"$ docker run -p 80:80 kennethreitz/httpbin"
}
}
}
}
}
}
},
"2":{
"tagName":"div",
"children":{
"0":{
"tagName":"div",
"children":{
"0":{
"tagName":"a",
"attributes":{
"href":"https://kennethreitz.org",
"target":"_blank"
},
"text":"the developer - Website"
}
}
},
"1":{
"tagName":"a",
"attributes":{
"href":"mailto:me@kennethreitz.org"
},
"text":"Send email to the developer"
}
}
}
}
},
"1":{
"tagName":"div",
"attributes":{
"class":"loading-container"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"loading"
}
}
}
}
}
}
}
}
}
}
}
}
}
},
"3":{
"tagName":"div",
"attributes":{
"class":"swagger-ui"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"wrapper"
},
"children":{
"0":{
"tagName":"section",
"attributes":{
"class":"clear"
},
"children":{
"0":{
"tagName":"span",
"attributes":{
"style":"float: right;"
},
"text":"[Powered by",
"children":{
"0":{
"tagName":"a",
"attributes":{
"target":"_blank",
"href":"https://github.com/rochacbruno/flasgger"
},
"text":"Flasgger"
},
"1":{
"tagName":"br"
}
}
}
}
}
}
}
}
},
"4":{
"tagName":"script",
"attributes":{
"src":"/flasgger_static/swagger-ui-bundle.js"
},
"text":" "
},
"5":{
"tagName":"script",
"attributes":{
"src":"/flasgger_static/swagger-ui-standalone-preset.js"
},
"text":" "
},
"6":{
"tagName":"script",
"attributes":{
"src":"/flasgger_static/lib/jquery.min.js",
"type":"text/javascript"
},
"text":""
},
"7":{
"tagName":"script",
"text":"\n\n window.onload = function () {\n \n\n fetch(\"/spec.json\")\n .then(function (response) {\n response.json()\n .then(function (json) {\n var current_protocol = window.location.protocol.slice(0, -1);\n if (json.schemes[0] != current_protocol) {\n // Switches scheme to the current in use\n var other_protocol = json.schemes[0];\n json.schemes[0] = current_protocol;\n json.schemes[1] = other_protocol;\n\n }\n json.host = window.location.host; // sets the current host\n\n const ui = SwaggerUIBundle({\n spec: json,\n validatorUrl: null,\n dom_id: '#swagger-ui',\n deepLinking: true,\n jsonEditor: true,\n docExpansion: \"none\",\n apisSorter: \"alpha\",\n //operationsSorter: \"alpha\",\n presets: [\n SwaggerUIBundle.presets.apis,\n // yay ES6 modules ↘\n Array.isArray(SwaggerUIStandalonePreset) ? SwaggerUIStandalonePreset : SwaggerUIStandalonePreset.default\n ],\n plugins: [\n SwaggerUIBundle.plugins.DownloadUrl\n ],\n \n // layout: \"StandaloneLayout\" // uncomment to enable the green top header\n })\n\n window.ui = ui\n\n // uncomment to rename the top brand if layout is enabled\n // $(\".topbar-wrapper .link span\").replaceWith(\"<span>httpbin</span>\");\n })\n })\n}\n "
},
"8":{
"tagName":"div",
"attributes":{
"class":"swagger-ui"
},
"children":{
"0":{
"tagName":"div",
"attributes":{
"class":"wrapper"
},
"children":{
"0":{
"tagName":"section",
"attributes":{
"class":"block col-12 block-desktop col-12-desktop"
},
"children":{
"0":{
"tagName":"div",
"children":{
"0":{
"tagName":"h2",
"text":"Other Utilities"
},
"1":{
"tagName":"ul",
"children":{
"0":{
"tagName":"li",
"children":{
"0":{
"tagName":"a",
"attributes":{
"href":"/forms/post"
},
"text":"HTML form"
}
}
}
}
},
"2":{
"tagName":"br"
},
"3":{
"tagName":"br"
}
}
}
}
}
}
}
}
}
}Last updated