const puppeteer = require("puppeteer");
const { Client } = require("pg");

const dbClient = new Client({
});

const { IncomingWebhook } = require("@slack/webhook");

const webhook = new IncomingWebhook(
  "https://hooks.slack.com/services/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"
);

const targetSelector = {
  searchBox: "#getJsonData",
  clearBtn: "#js_srcClearBtn",
  notFound: ".noResult",
  price: ".productPrice",
};

const limitNum = 10;

const searchProduct = async (PAGE, PRODUCT) => {
  const userAgentPuppeteer =
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.0 Safari/537.36";
  const userAgentNormal =
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36";
  const messages = [userAgentPuppeteer, userAgentNormal];
  const messageNum = Math.floor(Math.random() * messages.length);
  const page = await PAGE;
  const productArr = await PRODUCT;
  const URL = encodeURI(productArr);
  await page.setUserAgent(messages[messageNum]);
  await page.goto(`https://www.yodobashi.com?word=${URL}`);
};

const getLoopNum = async () => {
  const QUERY = "SELECT COUNT(id) FROM products";
  const loopNum = [];
  loopNum.splice(0);
  await dbClient
    .query(QUERY)
    .then((res) => {
      const indexAll = parseInt(res.rows[0].count);
      const index = Math.floor(indexAll / limitNum);
      const remainder = indexAll % limitNum;
      if (remainder > 0) {
        loopNum.push(index + 1);
      } else {
        loopNum.push(index);
      }
    })
    .catch((e) => console.error(e.stack));
  return loopNum[0];
};

const getProductName = async (LOOP) => {
  const loopNum = await LOOP;
  const productList = [];
  productList.splice(0);
  const getWord = "SELECT amazon_product_name , id FROM products LIMIT";
  for (let i = 0; i < loopNum; i++) {
    const offset = i * limitNum;
    const values = "'" + limitNum + "' OFFSET '" + offset + "';";
    const QUERY = getWord + values;
    await dbClient
      .query(QUERY)
      .then((res) => {
        const product = res.rows;
        productList.push(product);
      })
      .catch((e) => console.error(e.stack));
  }
  console.log("productList.length", productList.length);
  return productList;
};

const getPrice = async (PAGE, PRODUCT) => {
  const page = await PAGE;
  const productArr = await PRODUCT;
  await page.waitFor(2000);
  const notFound = await page.$(targetSelector.notFound);
  const catchPrice = await page.$(targetSelector.price);
    console.log("notFound=", notFound);
    console.log("catchPrice=", catchPrice);
    console.log("targetSelector.notFound=", targetSelector.notFound);
    console.log("targetSelector.price=", targetSelector.price);
  if (notFound) {
    console.log("notFound");
    const nothing =
      "UPDATE products SET yodobashi_product_price = " +
      null +
      " WHERE id = " +
      productArr.id +
      " ";
    dbClient.query(nothing, (err) => {
      if (err) {
        console.error(err.stack);
      }
    });
  } else if (catchPrice) {
    const priceList = await page.evaluate((selector) => {
      const list = Array.from(document.querySelectorAll(selector));
      return list.map((data) =>
        parseInt(data.textContent.slice(1).split(",").join(""))
      );
    }, targetSelector.price);
    const minPrice = function (a, b) {
      return Math.min(a, b);
    };
    console.log("priceList=", priceList);
    console.log(priceList.length > 0);
    console.log(
      "productArr.amazon_product_name",
      productArr.amazon_product_name
    );
    if (priceList.length > 0) {
      const price = priceList.reduce(minPrice);
      console.log("price=", price);
      const updatePrice =
        "UPDATE products SET yodobashi_product_price = " +
        price +
        " WHERE id = " +
        productArr.id +
        " ";
      dbClient.query(updatePrice, (err) => {
        if (err) {
          console.error(err.stack);
        }
      });
    } else {
      const soldOut =
        "UPDATE products SET yodobashi_product_price = " +
        0 +
        " WHERE id = " +
        productArr.id +
        " ";
      dbClient.query(soldOut, (err) => {
        if (err) {
          console.error(err.stack);
        }
      });
    }
  } else {
    console.log("getPriceはelse判定");
  }
  productArr = null;
};

const updateDB = async (PRODUCT, BROWSER) => {
  const productName = await PRODUCT;
  const browser = await BROWSER;
  const page = await browser.newPage();
  await page.waitFor(100);
  for (let i = 0; i < productName.length; i++) {
    console.log("for i=", i);
    // 下記3文はローカルと同様のため削除する。確認済み
    // console.log("productName[i].length=", productName[i].length);
    // console.log("productName[i]", productName[i]);
    // console.log("productName", productName);
    await Promise.all(
      productName[i].map(async (productArr) => {
        console.log("map i=", i);
        console.log("productArr", productArr);
        const page = await browser.newPage();
        await page.waitFor(100);
        await searchProduct(page, productArr.amazon_product_name).catch(() =>
          page.close()
        );
        await getPrice(page, productArr).catch(() => page.close());
        await page.close();
        productArr = null;
      })
    ).catch(() => {
      return null;
    });
    await productName[i].splice(0);
  }
};

const calculateDB = async () => {
  const QUERY =
    "UPDATE products SET comparison = amazon_product_price - yodobashi_product_price";
  await dbClient
    .query(QUERY)
    .then(() => {
      console.log("値段の比較が完了しました");
    })
    .catch((e) => console.error(e.stack));
};

const cleanUp = async (BROWSER) => {
  const browser = await BROWSER;
  await Promise.all([
    dbClient.on("drain", dbClient.end.bind(dbClient)),
    browser.close(),
  ]);
  dbClient.end((err) => {
    if (err) {
      console.error("DB切断時にエラーが発生しました", err.stack);
      process.exit(1);
    } else {
      console.log("DB切断しました");
      process.exit(0);
    }
  });
};

const notification = async (TEXT, CHANNEL, USERNAME) => {
  const notificationText = await TEXT;
  const channel = await CHANNEL;
  const username = await USERNAME;
  await webhook.send({
    text: notificationText,
    channel: channel,
    username: username,
  });
};

const main = async () => {
  try {
    await dbClient.connect();
    const loopIndex = await getLoopNum();
    const productName = await getProductName(loopIndex);
    const browser = await puppeteer.launch({
      headless: true,
      args: [
        "--no-sandbox",
        "--disable-setuid-sandbox",
        "-–disable-dev-shm-usage",
        "--disable-gpu",
        "--no-first-run",
        "--no-zygote",
        "--single-process",
      ],
    });
    await updateDB(productName, browser);
    await calculateDB();
    await notification(
      "値段比較に成功しました: https://data.heroku.com/dataclips/vfqjszrgbgkstplvsgkotjzpmdzi",
      "#ヨドバシ-値段比較",
      "ヨドバシ値段比較(成功)"
    );
    process.on("unhandledRejection", console.dir);
  } catch (e) {
    await notification(e, "#ヨドバシ-値段比較", "ヨドバシ値段比較(失敗)");
  } finally {
    const browser = await puppeteer.launch({
      headless: true,
      args: [
        "--no-sandbox",
        "--disable-setuid-sandbox",
        "-–disable-dev-shm-usage",
        "--disable-gpu",
        "--no-first-run",
        "--no-zygote",
        "--single-process",
      ],
    });
    await cleanUp(browser);
  }
};
main();
[cv:issue_marketplace_engineer]

const puppeteer = require("puppeteer");
const { Client } = require("pg");

const dbClient = new Client({
});

const { IncomingWebhook } = require("@slack/webhook");

const webhook = new IncomingWebhook(
  "https://hooks.slack.com/services/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"
);

const targetSelector = {
  searchBox: "#getJsonData",
  clearBtn: "#js_srcClearBtn",
  notFound: ".noResult",
  price: ".productPrice",
};

const limitNum = 10;

const searchProduct = async (PAGE, PRODUCT) => {
  const userAgentPuppeteer =
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.0 Safari/537.36";
  const userAgentNormal =
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36";
  const messages = [userAgentPuppeteer, userAgentNormal];
  const messageNum = Math.floor(Math.random() * messages.length);
  const page = await PAGE;
  const productArr = await PRODUCT;
  const URL = encodeURI(productArr);
  await page.setUserAgent(messages[messageNum]);
  await page.goto(`https://www.yodobashi.com?word=${URL}`);
};

const getLoopNum = async () => {
  const QUERY = "SELECT COUNT(id) FROM products";
  const loopNum = [];
  loopNum.splice(0);
  await dbClient
    .query(QUERY)
    .then((res) => {
      const indexAll = parseInt(res.rows[0].count);
      const index = Math.floor(indexAll / limitNum);
      const remainder = indexAll % limitNum;
      if (remainder > 0) {
        loopNum.push(index + 1);
      } else {
        loopNum.push(index);
      }
    })
    .catch((e) => console.error(e.stack));
  return loopNum[0];
};

const getProductName = async (LOOP) => {
  const loopNum = await LOOP;
  const productList = [];
  productList.splice(0);
  const getWord = "SELECT amazon_product_name , id FROM products LIMIT";
  for (let i = 0; i < loopNum; i++) {
    const offset = i * limitNum;
    const values = "'" + limitNum + "' OFFSET '" + offset + "';";
    const QUERY = getWord + values;
    await dbClient
      .query(QUERY)
      .then((res) => {
        const product = res.rows;
        productList.push(product);
      })
      .catch((e) => console.error(e.stack));
  }
  console.log("productList.length", productList.length);
  return productList;
};

const getPrice = async (PAGE, PRODUCT) => {
  const page = await PAGE;
  const productArr = await PRODUCT;
  await page.waitFor(2000);
  const notFound = await page.$(targetSelector.notFound);
  const catchPrice = await page.$(targetSelector.price);
    console.log("notFound=", notFound);
    console.log("catchPrice=", catchPrice);
    console.log("targetSelector.notFound=", targetSelector.notFound);
    console.log("targetSelector.price=", targetSelector.price);
  if (notFound) {
    console.log("notFound");
    const nothing =
      "UPDATE products SET yodobashi_product_price = " +
      null +
      " WHERE id = " +
      productArr.id +
      " ";
    dbClient.query(nothing, (err) => {
      if (err) {
        console.error(err.stack);
      }
    });
  } else if (catchPrice) {
    const priceList = await page.evaluate((selector) => {
      const list = Array.from(document.querySelectorAll(selector));
      return list.map((data) =>
        parseInt(data.textContent.slice(1).split(",").join(""))
      );
    }, targetSelector.price);
    const minPrice = function (a, b) {
      return Math.min(a, b);
    };
    console.log("priceList=", priceList);
    console.log(priceList.length > 0);
    console.log(
      "productArr.amazon_product_name",
      productArr.amazon_product_name
    );
    if (priceList.length > 0) {
      const price = priceList.reduce(minPrice);
      console.log("price=", price);
      const updatePrice =
        "UPDATE products SET yodobashi_product_price = " +
        price +
        " WHERE id = " +
        productArr.id +
        " ";
      dbClient.query(updatePrice, (err) => {
        if (err) {
          console.error(err.stack);
        }
      });
    } else {
      const soldOut =
        "UPDATE products SET yodobashi_product_price = " +
        0 +
        " WHERE id = " +
        productArr.id +
        " ";
      dbClient.query(soldOut, (err) => {
        if (err) {
          console.error(err.stack);
        }
      });
    }
  } else {
    console.log("getPriceはelse判定");
  }
  productArr = null;
};

const updateDB = async (PRODUCT, BROWSER) => {
  const productName = await PRODUCT;
  const browser = await BROWSER;
  const page = await browser.newPage();
  await page.waitFor(100);
  for (let i = 0; i < productName.length; i++) {
    console.log("for i=", i);
    // 下記3文はローカルと同様のため削除する。確認済み
    // console.log("productName[i].length=", productName[i].length);
    // console.log("productName[i]", productName[i]);
    // console.log("productName", productName);
    await Promise.all(
      productName[i].map(async (productArr) => {
        console.log("map i=", i);
        console.log("productArr", productArr);
        const page = await browser.newPage();
        await page.waitFor(100);
        await searchProduct(page, productArr.amazon_product_name).catch(() =>
          page.close()
        );
        await getPrice(page, productArr).catch(() => page.close());
        await page.close();
        productArr = null;
      })
    ).catch(() => {
      return null;
    });
    await productName[i].splice(0);
  }
};

const calculateDB = async () => {
  const QUERY =
    "UPDATE products SET comparison = amazon_product_price - yodobashi_product_price";
  await dbClient
    .query(QUERY)
    .then(() => {
      console.log("値段の比較が完了しました");
    })
    .catch((e) => console.error(e.stack));
};

const cleanUp = async (BROWSER) => {
  const browser = await BROWSER;
  await Promise.all([
    dbClient.on("drain", dbClient.end.bind(dbClient)),
    browser.close(),
  ]);
  dbClient.end((err) => {
    if (err) {
      console.error("DB切断時にエラーが発生しました", err.stack);
      process.exit(1);
    } else {
      console.log("DB切断しました");
      process.exit(0);
    }
  });
};

const notification = async (TEXT, CHANNEL, USERNAME) => {
  const notificationText = await TEXT;
  const channel = await CHANNEL;
  const username = await USERNAME;
  await webhook.send({
    text: notificationText,
    channel: channel,
    username: username,
  });
};

const main = async () => {
  try {
    await dbClient.connect();
    const loopIndex = await getLoopNum();
    const productName = await getProductName(loopIndex);
    const browser = await puppeteer.launch({
      headless: true,
      args: [
        "--no-sandbox",
        "--disable-setuid-sandbox",
        "-–disable-dev-shm-usage",
        "--disable-gpu",
        "--no-first-run",
        "--no-zygote",
        "--single-process",
      ],
    });
    await updateDB(productName, browser);
    await calculateDB();
    await notification(
      "値段比較に成功しました: https://data.heroku.com/dataclips/vfqjszrgbgkstplvsgkotjzpmdzi",
      "#ヨドバシ-値段比較",
      "ヨドバシ値段比較(成功)"
    );
    process.on("unhandledRejection", console.dir);
  } catch (e) {
    await notification(e, "#ヨドバシ-値段比較", "ヨドバシ値段比較(失敗)");
  } finally {
    const browser = await puppeteer.launch({
      headless: true,
      args: [
        "--no-sandbox",
        "--disable-setuid-sandbox",
        "-–disable-dev-shm-usage",
        "--disable-gpu",
        "--no-first-run",
        "--no-zygote",
        "--single-process",
      ],
    });
    await cleanUp(browser);
  }
};
main();
[![image](https://firebasestorage.googleapis.com/v0/b/issue-pro-42602.appspot.com/o/topics%2F17fa658b-53c7-4ca0-a28b-e588ed1bd371?alt=media&token=2c00afff-85ff-4428-8d15-696ab676dcb0)](https://i-ssue.com/lp/check_salary)

続きを読むには単価診断を受けてください（30秒）