Node.js (크롤링해온 정보(json 가공) react로 값 읽기)

---

참고자료

https://jung-story.tistory.com/100

Node.js (Node.js 를 이용한 웹 크롤링 하기 REST API )

---

개요

 

지난번에는 웹사이트에서 정보를 크롤링해오는 방법에 대해서 알아보았습니다.

 

이번에는 그 해당 정보를 여러개 가져오는 방법과 가져온 정보를 React Table 모듈에 가져오는 방법에 대해서 알아보도록 하겠습니다.

 

우선 저는 

http://ncov.mohw.go.kr/bdBoardList_Real.do?brdId=1&brdGubun=11&ncvContSeq=&contSeq=&board_id=&gubun=

이 사이트에서 값을 가져오도록 하겠습니다.

 

---

crawl.js

 

 

지난번과 다른점은 td > span인데 이것이 갖는 의미는 td속성 안의 span이라는 것입니다.

그리고 eq(0) , eq(2)와 같은 값이 있는데 이것은 해당하는 태그의 인덱스 값을 가져오는 기능을 해줍니다.

그리고 javascript 같은 경우는 return값을 배열로도 던져줄 수 있기에 여러 값을 배열을 통해서 return 해주도록 하였습니다.

 

---

 server.js

 

에서 만들어 놓은 함수들을 가져옵니다.

 

각각의 값들을 넣어주고..

 

이런 식으로 5000/api/crwal로 넘겨줍니다 json 형식으로

 

이와 같이 값을 불러올 수 있습니다.

 

---

/client/src/Posts.js

 

import React, { Component } from "react";
import { withStyles, makeStyles } from "@material-ui/core/styles";
import Table from "@material-ui/core/Table";
import TableBody from "@material-ui/core/TableBody";
import TableCell from "@material-ui/core/TableCell";
import TableContainer from "@material-ui/core/TableContainer";
import TableHead from "@material-ui/core/TableHead";
import TableRow from "@material-ui/core/TableRow";
import Paper from "@material-ui/core/Paper";
import { red } from "@material-ui/core/colors";

const StyledTableCell = withStyles((theme) => ({
  head: {
    backgroundColor: theme.palette.common.black,
    color: theme.palette.common.white,
  },
  body: {
    fontSize: 13,
    background:red
  },
}))(TableCell);

class Posts extends Component {
  /* 컴포넌트 생성시 */
  /* 생명주기순서 : constructor(생성자) -> componentWillMount -> render */
  constructor(props) {
    super(props);
    this.state = {
      crwal: [],
    };
  }
  componentWillMount() {
    fetch("api/crwal")
      .then((res) => res.json())
      .then((data) =>
        this.setState({
          crwal: data,
        })
      );
  }
  render() {
    const { crwal } = this.state;
    const crwallist = crwal.map((post) => (
      <div>
        <TableContainer component={Paper}>
          <Table>
            <TableHead>
              <TableRow>
                <StyledTableCell colSpan="4" align="center">
                  확진환자
                </StyledTableCell>
                <StyledTableCell colSpan="2" align="center">
                  사망자
                </StyledTableCell>
              </TableRow>
            </TableHead>
            <TableBody>
              <TableRow>
                <StyledTableCell align="center">누적</StyledTableCell>
                <StyledTableCell colSpan="3" align="center">
                  전일대비
                </StyledTableCell>
                <StyledTableCell align="center">누적</StyledTableCell>
                <StyledTableCell align="center">전일대비</StyledTableCell>
              </TableRow>
              <TableRow>
                <StyledTableCell align="center">누적확진자</StyledTableCell>
                <StyledTableCell align="center">소계</StyledTableCell>
                <StyledTableCell align="center">해외유입</StyledTableCell>
                <StyledTableCell align="center">국내발생</StyledTableCell>
                <StyledTableCell align="center" rowSpan="2">
                  {post.get2_accumulate}
                </StyledTableCell>
                <StyledTableCell align="center" rowSpan="2">
                  {post.get2_day}
                </StyledTableCell>
              </TableRow>
              <TableRow>
                <StyledTableCell align="center">
                  {post.accumulate}
                </StyledTableCell>
                <StyledTableCell align="center">{post.s_total}</StyledTableCell>
                <StyledTableCell align="center">
                  {post.Overseas}
                </StyledTableCell>
                <StyledTableCell align="center">
                  {post.domestic}
                </StyledTableCell>
              </TableRow>
            </TableBody>
          </Table>
        </TableContainer>
      </div>
    ));

    const agelist = crwal.map((post) => (
      <div>
        <TableContainer component={Paper}>
          <Table>
            <TableHead>
              <TableRow>
                <StyledTableCell align="center">구분</StyledTableCell>
                <StyledTableCell align="center">확진자</StyledTableCell>
                <StyledTableCell align="center">사망자</StyledTableCell>
                <StyledTableCell align="center">치명률</StyledTableCell>
              </TableRow>
            </TableHead>
            <TableBody>
              <TableRow>
                <StyledTableCell align="center">남성</StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_maccumulate}
                </StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_mdeath}
                </StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_mcritical}
                </StyledTableCell>
              </TableRow>
              <TableRow>
                <StyledTableCell align="center">여성</StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_waccumulate}
                </StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_wdeath}
                </StyledTableCell>
                <StyledTableCell align="center">
                  {post.get3_wcritical}
                </StyledTableCell>
              </TableRow>
            </TableBody>
          </Table>
        </TableContainer>
      </div>
    ));

    return (
      <div>
        <h2>※ 국내 발생 현황</h2>
        {crwallist}
        <p></p>
        <h2>※ 확진자 성별 현황</h2>
        {agelist}
      </div>
    );
  }
}
export default Posts;

 

---

crawl.js

 

const axios = require("axios");
const cheerio = require("cheerio");
const { text } = require("express");

let html = "";

async function getHtml() {
  try {
    return await axios.get(
      "http://ncov.mohw.go.kr/bdBoardList_Real.do?brdId=1&brdGubun=11&ncvContSeq=&contSeq=&board_id=&gubun="
    );
  } catch (error) {
    console.error(error);
  }
}

async function getNews() {
  if (!html) {
    html = await getHtml();
  }

  const $ = cheerio.load(html.data);
  let accumulate = {};
  $("#content .caseTable .ca_body li")
    .first("dl")
    .each(function (index, elem) {
      switch ($(this).find("dt").text().trim()) {
        case "누적":
          accumulate = $(this).find("dd").text();
          // .replace(/([\t|\n|\s])/gi, "");
          break;
      }
    });
  return accumulate;
}

async function getDay() {
  if (!html) {
    html = await getHtml();
  }

  const $ = cheerio.load(html.data);
  var s_total;
  var Overseas;
  var domestic;
  $("#content .caseTable .ca_body li .ca_value li")
    // .first("dl")
    .each(function (index, elem) {
      switch ($(this).find("strong").text().trim()) {
        case "소계":
          s_total = $(this).find("p").text();
          // .replace(/([\t|\n|\s])/gi, "");
          break;
        case "해외유입":
          Overseas = $(this).find("p").text();
        // .replace(/([\t|\n|\s])/gi, "");
        case "국내발생":
          domestic = $(this).find("p").text();
      }
    });
  return [s_total, Overseas, domestic];
}

async function getDay2() {
  if (!html) {
    html = await getHtml();
  }
  const $ = cheerio.load(html.data);
  var get2_accumulate;
  var get2_day;
  $("#content .caseTable .ca_body li").each(function (index, elem) {
    switch ($(this).find("dt").text().trim()) {
      case "누적":
        get2_accumulate = $(this).find("dd").text().trim();
      case "전일대비":
        get2_day = $(this).find("dd").text().trim();
    }
  });
  return [get2_accumulate, get2_day];
}

async function getDay3() {
  if (!html) {
    html = await getHtml();
  }
  const $ = cheerio.load(html.data);
  var get3_maccumulate;
  var get3_mdeath;
  var get3_mcritical;
  var get3_waccumulate;
  var get3_wdeath;
  var get3_wcritical;
  $("#content .data_table .num tr").each(function (index, elem) {
    switch ($(this).find("th").text().trim()) {
      case "남성":
        get3_maccumulate = $(this).find("td > span").eq(0).text().trim();
        get3_mdeath = $(this).find("td > span").eq(2).text().trim();
        get3_mcritical = $(this).find("td > span").eq(4).text().trim();
      case "여성":
        get3_waccumulate = $(this).find("td > span").eq(0).text().trim();
        get3_wdeath = $(this).find("td > span").eq(2).text().trim();
        get3_wcritical = $(this).find("td > span").eq(4).text().trim();
    }
  });
  return [
    get3_maccumulate,
    get3_mdeath,
    get3_mcritical,
    get3_waccumulate,
    get3_wdeath,
    get3_wcritical,
  ];
}

module.exports = { getNews, getDay, getDay2, getDay3 };

 

---

server.js

 

const express = require("express");
const { getNews, getDay, getDay2, getDay3 } = require("./crawl.js");
const bodyParser = require("body-parser");
const app = express();
const port = process.env.PORT || 5000;
const cron = require("node-cron");
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));

async function handleAsync() {
  const sum = await getNews();
  const day = await getDay();
  const day2 = await getDay2();
  const day3 = await getDay3();

  console.log(sum + "      " + day + "     " + day2, "     ", day3);
  return [sum, day, day2, day3];
}

cron.schedule("*/1 * * * *", async () => {
  console.log("running a task every two minutes");
  await handleAsync();
});

// app.use('/api/crwal',async(req,res) => {
//   const text = await handleAsync();
//   console.log(text);
//   res.json([{text: text},
//             {id : 1}]
//     );
// })

app.get("/api/crwal", async (req, res) => {
  const text = await handleAsync();
  var accumulate = text[0];
  var s_total = text[1][0];
  var Overseas = text[1][1];
  var domestic = text[1][2];
  var get2_accumulate = text[2][0];
  var get2_day = text[2][1];
  var get3_maccumulate = text[3][0];
  var get3_mdeath = text[3][1];
  var get3_mcritical = text[3][2];
  var get3_waccumulate = text[3][3];
  var get3_wdeath = text[3][4];
  var get3_wcritical = text[3][5];
  res.send([
    {
      id: 1,
      accumulate,
      s_total,
      Overseas,
      domestic,
      get2_accumulate,
      get2_day,
      get3_maccumulate,
      get3_mdeath,
      get3_mcritical,
      get3_waccumulate,
      get3_wdeath,
      get3_wcritical,
    },
  ]);
});

app.listen(port, () => console.log(`Listening on port ${port}`));

 

---

App.js

 

import React, { Component } from "react";
import Appbar from "./components/Appbar";
import Scroller from "./components/Scroller";
import { withStyles } from "@material-ui/core/styles";
import NewsCarousel from "./components/NewsCarousel";
import Posts from "./components/Posts";
const styles = (theme) => ({
  firstDiv: {
    backgroundColor: "#81c147",
    width: "100vw",
    height: "100vh",
  },
  secondDiv: {
    backgroundColor: "#1176a7",
    width: "100vw",
    height: "100vh",
  },
  thirdDiv: {
    backgroundColor: "#228f76",
    width: "100vw",
    height: "100vh",
  },
});

class App extends Component {
  render() {
    const { classes } = this.props;
    return (
      <div>
        <div id="section1" className={classes.firstDiv}>
          <Appbar />
          <Scroller />
          <Posts />
        </div>
        <div id="section2" className={classes.secondDiv}></div>
        <div id="section3" className={classes.thirdDiv}>
          <NewsCarousel />
        </div>
      </div>
    );
  }
}

export default withStyles(styles)(App); //withStyles 덕분에 firstDiv가 구별된다.