异步请求
将多个请求排队并立即接收task_id - 任务完成后,你可以使用所述task_id从请求中检索结果
对单个任务进行排队
单个查询或 url 端点- https://scraper-api.smartdaili-china.com/v1/data
使用你的首选参数向此端点发出 POST 请求,以在任务完成后接收用于检索的 task_id 以及使用的参数
curl -u username:password -X POST --url https://scraper-api.smartdaili-china.com/v2/data -H "Content-Type: application/json" -d "{\"url\": \"https://ip.smartproxy.com\", \"target\": \"universal\" }"
import requests
payload={
'target': 'universal',
'url': 'https://ip.smartproxy.com'
}
response = requests.request("POST", 'https://scraper-api.smartdaili-china.com/v1/data', auth=('user', 'pass'), data=payload)
print(response.text)
<?php
$params = array(
'url' => 'https://ip.smartproxy.com',
'target' => 'universal'
);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://scraper-api.smartdaili-china.com/v1/data');
curl_setopt($ch, CURLOPT_USERPWD, 'username' . ':' . 'password');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
$headers = array();
$headers[] = 'Content-Type: application/json';
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>
使用task_id 检索状态
单个查询或 url 端点 - https://scraper-api.smartdaili-china.com/v1/data/{task_id}
向此端点发出 GET 请求,并将 {task_id} 替换为从之前的 POST 请求收到的 ID,以检索用于任务的状态和参数
curl -u username:password https://scraper-api.smartdaili-china.com/v2/data/{task_id}
import requests
response = requests.request("GET", 'https://scraper-api.smartdaili-china.com/v1/data/{task_id}', auth=('user', 'pass'))
print(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://scraper-api.smartdaili-china.com/v1/data/{task_id}');
curl_setopt($ch, CURLOPT_USERPWD, 'username' . ':' . 'password');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>
使用task_id 检索状态
单个查询或 url 端点 - http://scrape.smartproxy.com/v1/data/{task_id}/results
向此端点发出 GET 请求,并将 {task_id} 替换为从之前的 POST 请求收到的 ID,以检索用于任务的状态和参数
curl -u username:password https://scraper-api.smartdaili-china.com/v2/data/{task_id}/results
import requests
response = requests.request("GET", 'https://scraper-api.smartdaili-china.com/v1/data/{task_id}/results', auth=('user', 'pass'))
print(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://scraper-api.smartdaili-china.com/v1/data/{task_id}/results');
curl_setopt($ch, CURLOPT_USERPWD, 'username' . ':' . 'password');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>
将多个任务放入队列
批次查询或 url 端点 - https://scraper-api.smartdaili-china.com/v1/data/batch
向此端点发出 POST 请求,以 JSON 格式提供多个查询或 URL
在单个批次中,你可以提交多个查询或 URL,但不能同时提交两者。此外,一批必须只有一个目标,如下面示例中所示的 google_search
import requests
import json
with open('queries.json', 'r') as f:
payload = json.loads(f.read())
response = requests.request(
'POST',
'https://scraper-api.smartdaili-china.com/v1/data/batch',
auth=('user', 'pass'),
json=payload,
)
print(response.text)
<?php
$ch = curl_init();
$payload = json_decode(file_get_contents('queries.json'), true);
curl_setopt($ch, CURLOPT_URL, 'https://scraper-api.smartdaili-china.com/v1/data/batch');
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_setopt($ch, CURLOPT_USERPWD, 'username:password');
$headers = array();
$headers[] = "Content-Type: application/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
$response = curl_exec($ch);
if ($response === false) {
echo 'Error: ' . curl_error($ch);
} else {
echo $response;
}
curl_close($ch);
?>
上面代码中使用的 queries.json 文件示例
{
"query":[
"blue",
"skyline",
"below"
],
"target": "google_search",
"parse": "true"
}
将任务状态接收到你的callback_url
通过输入callback_url 作为参数之一,这将适用于任何异步端点
任务完成后,我们将使用 task_id 和使用的参数向你提供的 URL 发出 POST 请求
你可以使用像这样的网站 来测试接收响应。
使用单个任务端点的示例:
curl -u username:password -X POST --url https://scraper-api.smartdaili-china.com/v2/data -H "Content-Type: application/json" -d "{\"url\": \"https://ip.smartproxy.com\", \"target\": \"universal\", \"callback_url\": \"<https://your.url>\" }"
import requests
payload={
'target': 'universal',
'url': 'https://ip.smartproxy.com',
'callback_url': '<https://your.url>'
}
response = requests.request("POST", 'https://scraper-api.smartdaili-china.com/v1/data', auth=('user', 'pass'), data=payload)
print(response.text)
<?php
$params = array(
'url' => 'https://ip.smartproxy.com',
'target' => 'universal',
'callback_url' => '<https://your.url>'
);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://scraper-api.smartdaili-china.com/v1/data');
curl_setopt($ch, CURLOPT_USERPWD, 'username' . ':' . 'password');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
$headers = array();
$headers[] = 'Content-Type: application/json';
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
$result = curl_exec($ch);
echo $result;
if (curl_errno($ch)) {
echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>
你将收到的回复示例
{
"id": "7039164056019693569",
"status": "done",
"target": "universal",
"query": "",
"url": "https://ip.smartproxy.com",
"domain": "com",
"num_pages": 10,
"locale": null,
"geo": null,
"device_type": "desktop",
"page_from": 1,
"parse": 0,
"output_schema": null,
"headless": null,
"priority": 0,
"persist": true,
"content_encoding": "utf-8",
"created_at": "2023-03-08 09:24:52",
"updated_at": "2023-03-08 09:24:52"
}
然后,你可以使用“id”通过此端点检索任务结果 –
https://scraper-api.smartdaili-china.com/v1/data/{task_id}/results
例如,要从上面的示例中检索结果,你可以将 GET 请求发送到:
https://scraper-api.smartdaili-china.com/v1/data/7039164056019693569/results
Updated 2 months ago