Node.js | 웹 크롤링 API 서버 만들기(Windows) | Selenium Crawling Server in Windows | 모듈화 하기

재사용성이 많은 함수들은 모듈로 만들어두면 매우 편하다. 예를 들면 아래처럼 driver 를 선언하는 함수는 각 라우터마다 동일하게 필요하다.

        // 웹드라이버 설정
        let driver = await new webdriver.Builder()
        .forBrowser('chrome')
        .setChromeOptions(new chrome.Options().windowSize({width: 1920, height: 1080}))
        .setChromeService(new chrome.ServiceBuilder(process.env.CHROMEDRIVER_PATH))
        .build();

 

또한 driver 메소드들을 우리 입맛에 맞게끔 만들어 두면 개발할 때도 훨씬 수월할 것이다.

 

참고로 selenium javacscript의 메소드에 관한 내용들은 아래 사이트에서 자세히 확인할 수 있다.

Index

 

The Selenium Browser Automation Project

1. modules 밑에 모듈을 내보내는 파일을 만든다.

 

/mdules/selenium/driverHandler.ts

import webdriver, { WebDriver } from 'selenium-webdriver';
import chrome from 'selenium-webdriver/chrome';
import firefox from 'selenium-webdriver/firefox'

// dotenv
import dotenv from 'dotenv'
dotenv.config();

// new driver 반환 함수
export const getDriverHandler = async () : Promise<WebDriver> => {
    const driver =  await new webdriver.Builder()
        .forBrowser('chrome')
        .setChromeOptions(new chrome.Options().windowSize({width: 1920, height: 1080}))
        .setChromeService(new chrome.ServiceBuilder(process.env.CHROMEDRIVER_PATH))
        .build();

    return driver;
}

 

/modules/index.ts

export * from "./db/expireHandler"

export * from "./selenium/driverHandler"

 

2. 라우터에서 모듈을 가져와 쓴다.

...

// modules
import { 
    getDriverHandler,
 } from '../modules';
 
 ...
 
// 웹드라이버 설정
let driver = await getDriverHandler();

 

이런식으로 여러개의 모듈을 만들 수 있다.

 

/modules/selenium/alertHandler.ts

import { until, WebDriver } from "selenium-webdriver";

// alert 확인 누르기
export const alertCloseAccept = async (driver : WebDriver ) => {

    // alert 창 뜰 때까지 기다림
    await driver.wait(until.alertIsPresent());

    // alert 로 드라이버 이동
    let alert = await driver.switchTo().alert();

    // alert text 사용하는 곳
    let alertText = await alert.getText();

    // 확인버튼 클릭
    await alert.accept();

    // 다시 원래 컨텐츠로 드라이버 이동
    await driver.switchTo().defaultContent();
}

// alert 닫기 누르기
export const alertCloseDismiss = async (driver : WebDriver ) => {

    // alert 창 뜰 때까지 기다림
    await driver.wait(until.alertIsPresent());

    // alert 로 드라이버 이동
    let alert = await driver.switchTo().alert();

    // alert text 사용하는 곳
    let alertText = await alert.getText();

    // 확인버튼 클릭
    await alert.dismiss();

    // 다시 원래 컨텐츠로 드라이버 이동
    await driver.switchTo().defaultContent();
}

// prompt 핸들러
export const promptCloseHandler = async (driver : WebDriver, text : string) => {

    // alert 창 뜰 때까지 기다림
    await driver.wait(until.alertIsPresent());

    // alert 로 드라이버 이동
    let alert = await driver.switchTo().alert();
    
    // prompt text 전송
    await alert.sendKeys(text)
    
    // 확인버튼 클릭
    await alert.accept();
    
    // 다시 원래 컨텐츠로 드라이버 이동
    await driver.switchTo().defaultContent(); 
}

 

/modules/selenium/navigationHandler.ts

import { WebDriver } from "selenium-webdriver";

export const naviGet = async ( driver : WebDriver, url : string) => {
    await driver.get(url);
}

export const naviBack = async (driver : WebDriver) => {
    await driver.navigate().back();
}

export const naviForward = async (driver : WebDriver) => {
    await driver.navigate().forward();
}

export const naviRefresh = async (driver : WebDriver) => {
    await driver.navigate().refresh();
}

 

/modules/selenium/cookieHandler.ts

import {  IWebDriverCookie, WebDriver } from "selenium-webdriver";

// 쿠키 추가하기
export const addCookie = async (driver : WebDriver, cookie : IWebDriverCookie ) => {
    await driver.manage().addCookie(cookie)
}

// 쿠키 하나 가져오기
export const getOneCookie = async (driver : WebDriver, key : string) : Promise<IWebDriverCookie> => {
    return await driver.manage().getCookie(key);
}

// 쿠키 전부 가져오기 
export const getAllCookie = async (driver : WebDriver) : Promise<IWebDriverCookie[]> => {
    return await driver.manage().getCookies();
}

// 쿠키 하나 지우기
export const deleteOneCookie = async (driver : WebDriver,  key : string) => {
    return await driver.manage().deleteCookie(key);
}

// 쿠키 전부 지우기
export const deleteAllCookie = async (driver : WebDriver) => {
    return await driver.manage().deleteAllCookies();
}

 

/modules/selenium/popupHandler.ts

import { WebDriver } from "selenium-webdriver";

export const popupClose = async (driver : WebDriver ) => {

    // window handles 0 빼고 모든 윈도우 닫음
    (await driver.getAllWindowHandles()).map(async (item : string, index: number)=> {
        if(index > 0) {
            await driver.switchTo().window((await driver.getAllWindowHandles())[index]);
            driver.close();
        }
    })

    // 메인 윈도우로 이동
    await driver.switchTo().window((await driver.getAllWindowHandles())[0]);

}

 

/modules/selenium/jqueryHandler.ts

import { By, WebDriver } from "selenium-webdriver";

// 해당 document의 value값을 바꿔준다.
export const JqChangeValueByID = async (driver : WebDriver, id : string, value: string ) => {
    await driver.executeScript(`document.getElementById('${id}').value='${value}';`)
}

export const JqSetAttribute = async (driver : WebDriver, id : string, attribute: string[] ) => {
    await driver.executeScript(`document.getElementById('${id}').setAttribute('${attribute[0]}','${attribute[1]}')`)
}

export const JqRemoveAttribute = async (driver : WebDriver, id : string, attribute: string ) => {
    await driver.executeScript(`document.getElementById('${id}').removeAttribute("${attribute}")`)
}

 

/modules/selenium/fileHandler.ts

import { Locator, until, WebDriver } from "selenium-webdriver";

// 경로에 맞는 파일 등록 ( 인증서 등 )
export const fileRegister = async (driver : WebDriver, func : Locator, path: string[] ) => { 
    let pathString : string = "";
    
    // 받은 배열을 줄바꿈을 포함한 문자열로 바꿔준다.
    path.map((item : string, index: number) => {
        if(index < path.length - 1) {
            pathString = pathString + item + " \n "
        }
        else {
            pathString = pathString + item
        }
    })
    
    const el = await driver.wait(until.elementLocated(func), 2000);
    return await driver.wait(until.elementIsVisible(el), 2000).sendKeys(pathString);

 

/modules/selenium/findHandler.ts

import { By, until, WebDriver } from "selenium-webdriver";

export const findElementById = async (driver: WebDriver, id: string, timeout = 2000) => {
    return await driver.wait(until.elementLocated(By.id(id)), timeout);
};
  
export const findElementByName = async (driver: WebDriver, name: string, timeout = 2000) => {
  return await driver.wait(until.elementLocated(By.name(name)), timeout);
};

export const findElementByCss = async (driver: WebDriver, css: string, timeout = 2000) => {
  return await driver.wait(until.elementLocated(By.css(css)), timeout);
};
  
export const findElementByXpath = async (driver: WebDriver, xpath: string, timeout = 2000) => {
  return await driver.wait(until.elementLocated(By.xpath(xpath)), timeout);
};

export const findElementByClass = async (driver: WebDriver, classname: string, timeout = 2000) => {
  return await driver.wait(until.elementLocated(By.className(classname)), timeout);
};

export const findElementByTagName = async (driver: WebDriver, tag: string, timeout = 2000) => {
  return await driver.wait(until.elementLocated(By.tagName(tag)), timeout);
};

export const findElementsById = async (driver: WebDriver, id: string) => {
  return  await driver.findElements(By.id(id));
};

export const findElementsByName = async (driver: WebDriver, name: string) => {
  return  await driver.findElements(By.name(name));
};

export const findElementsByCss = async (driver: WebDriver, css: string) => {
  return  await driver.findElements(By.css(css));
};

export const findElementsByXpath = async (driver: WebDriver, xpath: string) => {
  return  await driver.findElements(By.xpath(xpath));
};

export const findElementsByClass= async (driver: WebDriver, classname: string) => {
  return  await driver.findElements(By.className(classname));
};

export const findElementsByTagName= async (driver: WebDriver, tagName: string) => {
  return  await driver.findElements(By.tagName(tagName));
};

 

/modules/selenium/sleepHandler.ts

export const sleep = (ms : number) => {
    return new Promise((resolve) => setTimeout(resolve, ms));
}

 

 

/modules/index.ts

export * from "./db/expireHandler"

export * from "./selenium/alertHandler"
export * from "./selenium/cookieHandler"
export * from "./selenium/driverHandler"
export * from "./selenium/fileHandler"
export * from "./selenium/jqueryHandler"
export * from "./selenium/navigationHandler"
export * from "./selenium/popupHandler"
export * from "./selenium/findHandler"
export * from "./selenium/sleepHandler"