UlisesGascon
commented
6 years ago @Craftmeet Este sería el Script en Nodejs. Mañana lo comentamos en clase para entender un poco más el tema del scraping. En total tenemos 1326, quedaría retocar un poco algunos campos para hacerlo más interesante ;-)
const puppeteer = require('puppeteer'),
uuidv4 = require('uuid/v4'),
fs = require("fs");
function totalTime(origin){
const diffMs = (new Date() - origin); // milliseconds between now & origin
const diffDays = Math.floor(diffMs / 86400000);
const diffHrs = Math.floor((diffMs % 86400000) / 3600000);
const diffMins = Math.round(((diffMs % 86400000) % 3600000) / 60000);
return `Execution in total: ${diffDays} days, ${diffHrs} hours and ${diffMins} minutes`;
}
function pageScraper(pageNumber) {
return new Promise((resolve, reject) => {
let scrape = async () => {
console.log("[INFO][Artesanos][Scraper][Page] Just started page:", pageNumber);
const url = `http://www.juntadeandalucia.es/turismoydeporte/opencms/system/modules/com.saga.ctc.application/elements/registro-artesanos/jsonRegistro.jsp?idP=0&idM=0&idO=0&idT=0&pagina=${pageNumber}`;
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);
const result = await page.evaluate(() => {
const artesanosTabla = document.querySelectorAll("#report > tbody > tr");
let currentArtesano = 0;
const artesanos = [];
for (let i = 1; i <= artesanosTabla.length-1; i++) {
const elementoActual = artesanosTabla[i];
if(i % 2 !== 0) {
const bloqueTitulos = elementoActual.querySelectorAll("td");
artesanos[currentArtesano]= {
"Artesano": bloqueTitulos[1].innerText,
"Nombre comercial": bloqueTitulos[2].innerText,
"Provincia": bloqueTitulos[3].innerText
};
} else {
const bloqueDetalles = elementoActual.querySelectorAll("ul.no-margin > li");
for (let j = 1; j <= bloqueDetalles.length-1; j++) {
const textoRaw = bloqueDetalles[j].innerText.split(":");
artesanos[currentArtesano][textoRaw[0]] = textoRaw[1];
}
currentArtesano++;
}
}
return artesanos;
});
await browser.close();
return result;
};
scrape().then((values) => {
// Add UUID
values.map(item => {
item.uuid = uuidv4();
return item;
});
resolve(values)
console.log("[INFO][Artesanos][Scraper][Page] Just finished page:", pageNumber);
}).catch(err => {
console.log("[ERROR][Artesanos][Scraper][Page] Error:", pageNumber);
reject(err)
});
})
}
function scraper () {
console.log("[INFO][Artesanos][Scraper] Starting now....");
const startTime = new Date();
const paginationScraper = async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('http://www.juntadeandalucia.es/turismoydeporte/opencms/system/modules/com.saga.ctc.application/elements/registro-artesanos/jsonRegistro.jsp');
const result = await page.evaluate(() => {
return document.querySelectorAll(".pagination > li").length;
});
await browser.close();
return result;
};
paginationScraper().then((totalPages) => {
console.log("[INFO][Artesanos][Scraper] Total pages to scrap for pagination:", totalPages);
let paginationData = [];
(async () => {
for(let i = 1; i <= totalPages; i++){
let page = await pageScraper(i);
paginationData.push(page);
}
const cleanData = [].concat(...paginationData);
fs.writeFileSync(`${__dirname}/data/artesanos.json`, JSON.stringify(cleanData));
console.log("[INFO][Artesanos][Scraper] Ended sucesfully. All data is now in the database");
console.log("[INFO][Artesanos][Scraper] Ended sucesfully. All data is now in the database");
console.log("[INFO][Artesanos][Scraper]", totalTime(startTime));
})();
}).catch(console.log);
}
scraper();
Craftmeet
Duda sobre Scraping