Created
June 5, 2021 17:04
-
-
Save vladgovor77771/3f0283b9c0b3720ce4d9a6cab2354205 to your computer and use it in GitHub Desktop.
Cloudflare: Curl vs node-libcurl vs axios vs node-fetch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const logger = console; | |
| const main = async () => { | |
| logger.info("Testing curl exec"); | |
| await testCurlExec(); | |
| logger.info("Testing node-libcurl"); | |
| await testNodeLibCurl(); | |
| logger.info("Testing axios"); | |
| await testAxios(); | |
| logger.info("Testing node-fetch"); | |
| await testNodeFetch(); | |
| }; | |
| const isGood = data => { | |
| return data.indexOf(`<script src="/static/js/main.`) > -1; | |
| }; | |
| const testCurlExec = async () => { | |
| const util = require("util"); | |
| const exec = util.promisify(require("child_process").exec); | |
| try { | |
| let command = | |
| "curl " + | |
| '--header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" ' + | |
| '--header "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3" ' + | |
| '--header "Connection: close" ' + | |
| '--header "Te: trailers" ' + | |
| '--header "Upgrade-Insecure-Requests: 1" ' + | |
| '--user-agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0" ' + | |
| "https://all-access.wax.io/"; | |
| const { stdout } = await exec(command); | |
| // logger.debug(stdout); | |
| logger.info(isGood(stdout) ? "Page good" : "Page bad"); | |
| } catch (err) { | |
| logger.error("Can't execute curl: " + err); | |
| } | |
| }; | |
| const testNodeLibCurl = async () => { | |
| try { | |
| const { curly } = require("node-libcurl"); | |
| const res = await curly.get("https://all-access.wax.io/", { | |
| userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", | |
| httpHeader: [ | |
| "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", | |
| "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3", | |
| "Upgrade-Insecure-Requests: 1", | |
| "Te: trailers", | |
| "Upgrade-Insecure-Requests: 1", | |
| "Connection: close", | |
| ], | |
| // acceptEncoding: "gzip, deflate", // doesn't matter | |
| sslVerifyPeer: false, // if true also doesn't work | |
| }); | |
| // logger.debug(res.data); // error code: 1020 | |
| logger.info(isGood(res.data) ? "Page good" : "Page bad"); | |
| } catch (err) { | |
| logger.error("Curly error: " + err); | |
| } | |
| }; | |
| const testAxios = async () => { | |
| const axios = require("axios"); | |
| try { | |
| let res = await axios({ | |
| method: "GET", | |
| url: "https://all-access.wax.io/", | |
| headers: { | |
| Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", | |
| "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3", | |
| "Upgrade-Insecure-Requests": "1", | |
| Te: "trailers", | |
| "Upgrade-Insecure-Requests": "1", | |
| Connection: "close", | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", | |
| }, | |
| }); | |
| // logger.debug(res.data); // error code: 1020 | |
| logger.info(isGood(res.data) ? "Page good" : "Page bad"); | |
| } catch (err) { | |
| logger.error("axios error: " + err); | |
| } | |
| }; | |
| const testNodeFetch = async () => { | |
| const fetch = require("node-fetch"); | |
| try { | |
| let res = await fetch("https://all-access.wax.io/", { | |
| method: "GET", | |
| headers: { | |
| Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", | |
| "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3", | |
| "Upgrade-Insecure-Requests": "1", | |
| Te: "trailers", | |
| "Upgrade-Insecure-Requests": "1", | |
| Connection: "close", | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0", | |
| }, | |
| }); | |
| let text = await res.text(); | |
| // logger.debug(text); // error code: 1020 | |
| logger.info(isGood(text) ? "Page good" : "Page bad"); | |
| } catch (err) { | |
| logger.error("node-fetch: " + err); | |
| } | |
| }; | |
| main(); |
I'm trying to fetch a page protected by CloudFlare and found your script. did you ever get this to work?
@ekkis
hey, I think CloudFlare looks on tls version and user-agent, also it looks on your ip (use good proxies)
I have no idea how to bypass cf from Node.js, but
Try firstly request from linux default curl app, if there is no 1020 error, then it is most likely real to bypass, otherwise I'm not sure
Try to use httpx python library
If you can use rust, try reqwest library with rustls
For me, both variants works
thanks for the reply. I found that Puppeteer works right through CF so I've implemented a solution that uses it
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I'm trying to fetch a page protected by CloudFlare and found your script. did you ever get this to work?