This tutorial will look at utilizing Puppeteer to automate Table Data Scraping into JSON. An open-source Node library called Puppeteer offers a high-level API for managing browsers based on Chrome or Chromium via the DevTools Protocol. With Puppeteer, you can automate any task that you can carry out using a Chrome browser. A puppeteer is thus a perfect tool for test automation and site scraping. Everything you need to know about using Puppeteer to automate form submission will be covered in this post. We will talk about it. <ul><li>Automated Table Data Scraping</li><li>Using Puppeteer selectors to target table data</li></ul> <pre class="ql-syntax" spellcheck="false">const puppeteer = require("puppeteer");
const fs = require("fs");

async function scraper (option){
&nbsp; try {
&nbsp;&nbsp; //browser initiated
&nbsp;&nbsp; let browser = await puppeteer.launch({
&nbsp;&nbsp;&nbsp; args: ["--start-maximized"],
&nbsp;&nbsp;&nbsp; headless: false,
&nbsp; });
&nbsp; // opens a new blank page
&nbsp; let page = await browser.newPage();

&nbsp; // resize the browser
&nbsp; await page.setViewport({ width: 1366, height: 768 });

&nbsp; // navigate to url and wait until page loads completely
&nbsp; await page.goto(`https://www.publicsurplus.com/sms/ufl,fl/list/current?slth=&amp;sma=&amp;orgid=1239&amp;sorg=&amp;ctxnId=341800115&amp;page=${option.code}&amp;sortBy=timeLeft`, {
&nbsp;&nbsp;&nbsp; waitUntil: "networkidle0",
&nbsp; });

//wait for the page to load
&nbsp;&nbsp;&nbsp; await page.waitForSelector(".tabCurr");

&nbsp;&nbsp;&nbsp; const content = await page.evaluate(() =&gt; {

&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; const tbody = document.querySelector(
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ".tabCurr tbody",
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; );

// iterate through the table rows
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; const trs = Array.from(
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; tbody.querySelectorAll("tr"),
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; );

&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; const content = [];

// iterate through each row of table
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; for (const tr of trs) {
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; const tds = Array.from(tr.querySelectorAll("td"));
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; const data = tds.map((td) =&gt; td.innerText);
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; &nbsp;
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if (tds.length &gt;= 5) {
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; // push the data
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; content.push({
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; auction: data[0],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; title: data[1],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; image: data[2],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; time_left: data[3],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; current_price: data[4],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; notes: data[5],
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; });
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; &nbsp;
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }
&nbsp;&nbsp;&nbsp; &nbsp;
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return content;
&nbsp;&nbsp;&nbsp; });

// writes data into JSON file
&nbsp;&nbsp; fs.writeFile(
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; `./Data/Data-${option.index}.json`,
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; JSON.stringify(content, null, 2),
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; (err) =&gt; {
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; if (err) {
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; console.log(err);
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; } else {
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; console.log(`Data&nbsp; of&nbsp; Page-${option.index} Scraped`);
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; },
&nbsp;&nbsp;&nbsp; );

&nbsp;&nbsp;&nbsp; await browser.close();
&nbsp; } catch (e) {
&nbsp;&nbsp;&nbsp; console.log(e);
&nbsp; }
}



let main = async () =&gt; {
&nbsp; // Loop through the page 
// increase the iteration with the number of pages to scrape
&nbsp; for (let i = 0; i &lt;= 1; i++) {
&nbsp;&nbsp;&nbsp; await Promise.all([
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; scraper({
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; code: i,
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; index: i,
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }),
&nbsp;&nbsp;&nbsp; ]);
&nbsp; }
};

main()
&nbsp; .then(() =&gt; {
&nbsp;&nbsp;&nbsp; console.log("Scraping Completed!");
&nbsp; })
&nbsp; .catch((e) =&gt; {
&nbsp;&nbsp;&nbsp; console.log(`Failed due to exception - ${e}`);
&nbsp; });
</pre> I have commented on each and every line that is important. If you want further assistance contact me. Happy coding ...

Scraping Table Data into JSON File with Puppeteer

Related blogs

About us: