Node.js를 이용한 파일 읽기 및 쓰기, 웹 크롤링

index.js

const http = require('http');
const express = require('express');
const app = express();

const fs = require('fs');

const axios = require('axios');
const cheerio = require('cheerio');
// axios 한글 깨짐 해결 하는 모듈
const iconv = require('iconv-lite');

app.set('port', 3000);

const sleep = (ms) => {
    return new Promise(resolve=>{
        setTimeout(resolve,ms)
    })
}
app.get('/axios2', (req, res) => {
    // Promise - 콜백 헬에 빠지는것을 방지(흐름제어) - 메소드체인.then([콜백])
    // Async - 리스트 형식으로 한다. [콜백, 콜백, 콜백 ...]
    let getUrlVal = "URL"; // URL 입력
    axios.get(getUrlVal, {responseType:"arraybuffer"}).then(async(response) => {
        const htmlContent = response.data;
        let htmlCMD = iconv.decode(htmlContent,"EUC-KR").toString();
        // cheerio를 이용한 DOM셀렉터
        const $ = cheerio.load(htmlCMD);
        //#main_content > div > div._persist > div:nth-child(1) > div:nth-child(4) > div.cluster_body > ul > li:nth-child(1) > div.cluster_thumb > div > a > img
        let imgData = $('ul > li > div.cluster_thumb > div > a > img');
        for(var i=0, cnt=0; i<10; i++) {
            let imgUrl = imgData[i].attribs.src
            //console.log(imgUrl.split('?')[0]);
            let imgDataUrl = imgUrl.split('?')[0];
            //console.log(imgDataUrl);
            axios.get(imgDataUrl, {responseType: 'arraybuffer'}).then( (imgRes)=>{
                //console.log(imgRes.data);
                fs.writeFile("./download/"+cnt+".jpg", imgRes.data, (err, data1)=>{
                    console.log(">>> 다운로드 완료 " + cnt++);
                });
            });
            await sleep(100);
        }
    });
    res.end();
});

app.get('/', (req, res)=>{
    res.end('<h1>Hello nodejs world</h1>')
});

app.get('/fs', (req, res)=>{
    let message = req.query.message;

    fs.writeFile('test.txt', message, function (err) {
        if (err) throw err;
        console.log('Saved!');
        res.redirect('/fs_read');
      });
});

app.get('/fs_read', (req, res)=>{
    fs.readFile('test.txt', function(err, data) {
        res.writeHead(200, {'Content-Type': 'text/html; charset=UTF-8'});
        res.write(`<h3>${data}</h3>`);
        res.end('<h1>파일 읽기 테스트</h1>')
      });   
});

const server = http.createServer(app);
server.listen(app.get('port'), ()=>{
    console.log('서버 실행 중: http://localhost:' + app.get('port'));
});

 

package.json

{
  "name": "openapi_workspace2022",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "start": "nodemon index.js",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "axios": "^0.27.2",
    "cheerio": "^1.0.0-rc.12",
    "express": "^4.18.1",
    "iconv-lite": "^0.6.3"
  },
  "devDependencies": {
    "nodemon": "^2.0.19"
  }
}