spox-ripper/index.js

const request = require("request").defaults({ jar: true });
const cheerio = require("cheerio");
const fs = require("fs");

const spoxurl = "http://asd.spox.spoj.pl:81";
const loginurl = spoxurl + "/account/login/";
const lessonsurl = spoxurl + "/LO1516/units/";

const username = "nick";
const password = "password";

function getToken() {
  return new Promise((done, reject) => {
    let data = "";
    let crfToken = "";
    request.get(spoxurl).on("response", res => {
      crfToken = res.headers["set-cookie"][0]
        .split(";")
        .shift()
        .split("=")
        .pop();
      done(crfToken);
    }); //.on("data", deltaData => {
    // data += deltaData;
    // }).on("end", ()=>{
    // console.log(data);
    // });
  });
}

function login(token) {
  return new Promise((resolve, reject) => {
    request
      .post(loginurl)
      .form({
        username: username,
        password: password,
        csrfmiddlewaretoken: token,
        next: ""
      })
      .on("response", response => {
        resolve();
      });
  });
}

function fetchLessons() {
  return new Promise((resolve, reject) => {
    let body = "";
    request
      .get(lessonsurl)
      .on("data", bodyDelta => {
        body += bodyDelta;
      })
      .on("end", () => {
        let $ = cheerio.load(body);
        let units = $(".section-body > div.edit-section-link");
        fs.writeFile("temp.html", body, () => {});
        console.log(units.length);
        let temp = [];
        units.each((i, el) => {
          let a = $(el).find("a");
          // console.log(a.attr("href"), a.text());
          temp.push({
            link: a.attr("href"),
            title: a.text()
          });
        });
        // units.forEach((el) => {
        //     console.log(el.attr("href"));
        // });
        resolve(temp);
      });
  });
}

function predownload(link) {
  return new Promise((resolve, reject) => {
    let body = "";
    console.log(spoxurl + link + "/");
    request
      .get(spoxurl + link + "/")
      .on("data", delta => (body += delta))
      .on("end", () => {
        let $ = cheerio.load(body);
        let link = $("table a").attr("href");
        console.log(link);
        resolve(link);
      });
  });
}

function download(link) {
  return new Promise((resolve, reject) => {
    if (!link) {
      resolve();
      return;
    }
    let body = "";
    request
      .get(spoxurl + link)
      .on("data", delta => (body += delta))
      .on("end", () => {
        let $ = cheerio.load(body);
        let problembody = $("div.section-body.problem-body").text();
        let code = $("textarea#id_source").val();
        let title = $("section header h1")
          .text()
          .toLocaleLowerCase()
          .replace(/ /g, "_");
        if (!fs.existsSync("./output/" + title)) {
          fs.mkdirSync("./output/" + title);
        }
        fs.writeFile(
          "./output/" + title + "/problem.txt",
          problembody,
          () => {}
        );
        fs.writeFile("./output/" + title + "/code.c", code, () => {});
        fs.writeFile("./output/" + title + "/raw.html", body, () => {});
        resolve();
      });
  });
}

getToken()
  .then(login)
  .then(fetchLessons)
  .then(arr => {
    let all = [];
    arr.forEach(o => {
      all.push(predownload(o.link).then(download));
    });
    Promise.all(all);
  })
  .catch(error => {
    console.log(error);
  });