Added New Scraper

MAYKELL07 · Nov 5, 2023 · 0765460 · 0765460
1 parent d1d73a8
commit 0765460
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 13 deletions.
diff --git a/postcss.config.js b/postcss.config.js
@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}
diff --git a/src/client/pages/home.tsx b/src/client/pages/home.tsx
@@ -1,5 +1,6 @@
 import { useState } from 'react';
 import axios from 'axios';
+import '../../../styles/global.css';
 
 const Home = () => {
   const [formData, setFormData] = useState({

diff --git a/src/server/scraper/scraper.controller.ts b/src/server/scraper/scraper.controller.ts
@@ -1,5 +1,6 @@
-import { Controller, Post, Body } from '@nestjs/common';
+import { Controller, Post, Body, Get, Res } from '@nestjs/common';
 import { ScraperService } from './scraper.service';
+import { Response } from 'express';
 
 @Controller('scraper')
 export class ScraperController {
@@ -22,4 +23,15 @@ export class ScraperController {
     async scrapeGenshinStats(@Body('userUID') userUID: string): Promise<any> {
         return await this.scraperService.scrapeGenshinStats(userUID);
     }
+
+    @Get('indonesia-stock-data')
+    async getIndonesiaStockData(@Res() res: Response) {
+        try {
+            const screenshotBuffer = await this.scraperService.scrapeIndonesiaStockData();
+            res.setHeader('Content-Type', 'image/png');
+            res.send(screenshotBuffer);
+        } catch (error) {
+            res.status(500).send('Failed to capture screenshot');
+        }
+    }
 }
diff --git a/src/server/scraper/scraper.service.ts b/src/server/scraper/scraper.service.ts
@@ -10,7 +10,7 @@ export class ScraperService implements OnModuleInit, OnModuleDestroy {
 
     async onModuleInit() {
         const playwright = addExtra(require('playwright')).use(StealthPlugin());
-        this.browser = await playwright.chromium.launch({ headless: true });
+        this.browser = await playwright.chromium.launch({ headless: false });
         this.cookieData = await fs.readFile('./src/server/scraper/cookie.json', 'utf8');
     }
 
@@ -49,18 +49,18 @@ export class ScraperService implements OnModuleInit, OnModuleDestroy {
         });
         const page = await context.newPage();
         await page.goto('https://www.alodokter.com', { waitUntil: 'domcontentloaded' });
-    
+
         await page.fill('#searchinput', inputText);
         await page.keyboard.press('Enter');
-        
+
         try {
             await page.waitForSelector('#articles-result > card-post-index:nth-child(1)', { state: 'attached', timeout: 5000 });
         } catch (error) {
             console.error('Element not found:', error);
             await context.close();
             return null;
         }
-    
+
         const result = await page.evaluate(() => {
             const card = document.querySelector('#articles-result > card-post-index:nth-child(1)');
             if (card) {
@@ -74,12 +74,12 @@ export class ScraperService implements OnModuleInit, OnModuleDestroy {
                 return null;
             }
         });
-    
+
         // Prepend "https://www.alodokter.com" to the link
         if (result && result.link) {
             result.link = `https://www.alodokter.com${result.link}`;
         }
-    
+
         await context.close();
         return result;
     }
@@ -90,25 +90,25 @@ export class ScraperService implements OnModuleInit, OnModuleDestroy {
         });
         const page = await context.newPage();
         await page.goto('https://dak.gg/genshin/en');
-    
+
         // Input the UID
         await page.fill('#__next > div > main > div > div.container > div.search-input > div > form > input[type=text]', userUID);
         await page.keyboard.press('Enter');
-    
+
         // Capture screenshot of the chart
         /* const chartElement = await page.waitForSelector('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.chart');
         await chartElement.screenshot({ path: 'genshin_chart.png', animations: 'disabled' }); */
-    
+
         // Scrape the numerical values
         const adventureRank = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(1) > div:nth-child(3)');
         const worldLevel = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(2) > div:nth-child(3)');
         const daysActive = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(3) > div:nth-child(3)');
         const achievements = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(4) > div:nth-child(3)');
         const spiralAbyss = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(5) > div:nth-child(3)');
         const spiralAbyssBattle = await page.textContent('#__next > div > main > div > div.contents-holder > div > div.sc-c1da52f4-0.loONIv > div.sc-e2832524-0.cBZWkv > div.detail > div.contents > div.sc-4cd95f9a-0.kIECDb.en > div:nth-child(5) > div.battle-time');
-        
+
         await context.close();
-        
+
         return {
             adventureRank,
             worldLevel,
@@ -118,5 +118,25 @@ export class ScraperService implements OnModuleInit, OnModuleDestroy {
             spiralAbyssBattle
         };
     }
-
+
+    async scrapeIndonesiaStockData(): Promise<Buffer> {
+        const context = await this.browser.newContext({
+            userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
+        });
+        const page = await context.newPage();
+        await page.goto('https://www.idx.co.id/id/data-pasar/ringkasan-perdagangan/ringkasan-saham', { waitUntil: 'networkidle' });
+
+        const elementLocator = page.locator('#app > div.sticky-footer-container-item.--pushed > main > div > div.vgt-boxed.mb-24.vgt-wrap > div > div.vgt-responsive');
+
+        await elementLocator.waitFor();
+
+        const overlayButton = page.locator('#app > div:nth-child(5) > a');
+        await overlayButton.evaluate(node => node.style.display = 'none');
+
+        const screenshotBuffer = await elementLocator.screenshot();
+
+        await context.close();
+        return screenshotBuffer;
+    }
+
 }
diff --git a/src/client/pages/home.css → styles/global.css b/src/client/pages/home.css → styles/global.css