-
Notifications
You must be signed in to change notification settings - Fork 0
/
LinkGrabber.js
45 lines (37 loc) · 1.03 KB
/
LinkGrabber.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
const axios = require('axios');
var cheerio = require('cheerio');
async function flareSolverr(url) {
var data = JSON.stringify({
"cmd": "request.get",
"url": url,
"maxTimeout": 120000
});
var config = {
method: 'post',
url: 'http://127.0.01:8191/v1',
headers: {
'Content-Type': 'application/json'
},
data: data
};
res = await axios(config)
return res.data.solution.response
}
async function getLinksFromURL(url) {
try {
let links = [];
let scrape = await flareSolverr(url)
// let httpResponse = await axios.get(url);
let $ = cheerio.load(scrape);
let linkObjects = $('a'); // get all hyperlinks
linkObjects.each((index, element) => {
links.push(
$(element).attr('href'), // get the href attribute
);
});
return links;
} catch (e) { console.log(e) }
}
module.exports = {
getLinksFromURL
}