const request = require("request").defaults({ jar: true }); const cheerio = require("cheerio"); const fs = require("fs"); const spoxurl = "http://asd.spox.spoj.pl:81"; const loginurl = spoxurl + "/account/login/"; const lessonsurl = spoxurl + "/LO1516/units/"; const username = "nick"; const password = "password"; function getToken() { return new Promise((done, reject) => { let data = ""; let crfToken = ""; request.get(spoxurl).on("response", res => { crfToken = res.headers["set-cookie"][0] .split(";") .shift() .split("=") .pop(); done(crfToken); }); //.on("data", deltaData => { // data += deltaData; // }).on("end", ()=>{ // console.log(data); // }); }); } function login(token) { return new Promise((resolve, reject) => { request .post(loginurl) .form({ username: username, password: password, csrfmiddlewaretoken: token, next: "" }) .on("response", response => { resolve(); }); }); } function fetchLessons() { return new Promise((resolve, reject) => { let body = ""; request .get(lessonsurl) .on("data", bodyDelta => { body += bodyDelta; }) .on("end", () => { let $ = cheerio.load(body); let units = $(".section-body > div.edit-section-link"); fs.writeFile("temp.html", body, () => {}); console.log(units.length); let temp = []; units.each((i, el) => { let a = $(el).find("a"); // console.log(a.attr("href"), a.text()); temp.push({ link: a.attr("href"), title: a.text() }); }); // units.forEach((el) => { // console.log(el.attr("href")); // }); resolve(temp); }); }); } function predownload(link) { return new Promise((resolve, reject) => { let body = ""; console.log(spoxurl + link + "/"); request .get(spoxurl + link + "/") .on("data", delta => (body += delta)) .on("end", () => { let $ = cheerio.load(body); let link = $("table a").attr("href"); console.log(link); resolve(link); }); }); } function download(link) { return new Promise((resolve, reject) => { if (!link) { resolve(); return; } let body = ""; request .get(spoxurl + link) .on("data", delta => (body += delta)) .on("end", () => { let $ = cheerio.load(body); let problembody = $("div.section-body.problem-body").text(); let code = $("textarea#id_source").val(); let title = $("section header h1") .text() .toLocaleLowerCase() .replace(/ /g, "_"); if (!fs.existsSync("./output/" + title)) { fs.mkdirSync("./output/" + title); } fs.writeFile( "./output/" + title + "/problem.txt", problembody, () => {} ); fs.writeFile("./output/" + title + "/code.c", code, () => {}); fs.writeFile("./output/" + title + "/raw.html", body, () => {}); resolve(); }); }); } getToken() .then(login) .then(fetchLessons) .then(arr => { let all = []; arr.forEach(o => { all.push(predownload(o.link).then(download)); }); Promise.all(all); }) .catch(error => { console.log(error); });