const puppeteer = require("puppeteer");
const { Client } = require("pg");

const dbClient = new Client({
});

const { IncomingWebhook } = require("@slack/webhook");

const webhook = new IncomingWebhook(
"https://hooks.slack.com/services/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"
);

const targetSelector = {
searchBox: "#getJsonData",
clearBtn: "#js_srcClearBtn",
notFound: ".noResult",
price: ".productPrice",
};

const limitNum = 10;

const searchProduct = async (PAGE, PRODUCT) => {
const userAgentPuppeteer =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.0 Safari/537.36";
const userAgentNormal =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36";
const messages = [userAgentPuppeteer, userAgentNormal];
const messageNum = Math.floor(Math.random() * messages.length);
const page = await PAGE;
const productArr = await PRODUCT;
const URL = encodeURI(productArr);
await page.setUserAgent(messages[messageNum]);
await page.goto(https://www.yodobashi.com?word=${URL});
};

const getLoopNum = async () => {
const QUERY = "SELECT COUNT(id) FROM products";
const loopNum = [];
loopNum.splice(0);
await dbClient
.query(QUERY)
.then((res) => {
const indexAll = parseInt(res.rows[0].count);
const index = Math.floor(indexAll / limitNum);
const remainder = indexAll % limitNum;
if (remainder > 0) {
loopNum.push(index + 1);
} else {
loopNum.push(index);
}
})
.catch((e) => console.error(e.stack));
return loopNum[0];
};

const getProductName = async (LOOP) => {
const loopNum = await LOOP;
const productList = [];
productList.splice(0);
const getWord = "SELECT amazon_product_name , id FROM products LIMIT";
for (let i = 0; i < loopNum; i++) {
const offset = i * limitNum;
const values = "'" + limitNum + "' OFFSET '" + offset + "';";
const QUERY = getWord + values;
await dbClient
.query(QUERY)
.then((res) => {
const product = res.rows;
productList.push(product);
})
.catch((e) => console.error(e.stack));
}
console.log("productList.length", productList.length);
return productList;
};

const getPrice = async (PAGE, PRODUCT) => {
const page = await PAGE;
const productArr = await PRODUCT;
await page.waitFor(2000);
const notFound = await page.$(targetSelector.notFound);
const catchPrice = await page.$(targetSelector.price);
console.log("notFound=", notFound);
console.log("catchPrice=", catchPrice);
console.log("targetSelector.notFound=", targetSelector.notFound);
console.log("targetSelector.price=", targetSelector.price);
if (notFound) {
console.log("notFound");
const nothing =
"UPDATE products SET yodobashi_product_price = " +
null +
" WHERE id = " +
productArr.id +
" ";
dbClient.query(nothing, (err) => {
if (err) {
console.error(err.stack);
}
});
} else if (catchPrice) {
const priceList = await page.evaluate((selector) => {
const list = Array.from(document.querySelectorAll(selector));
return list.map((data) =>
parseInt(data.textContent.slice(1).split(",").join(""))
);
}, targetSelector.price);
const minPrice = function (a, b) {
return Math.min(a, b);
};
console.log("priceList=", priceList);
console.log(priceList.length > 0);
console.log(
"productArr.amazon_product_name",
productArr.amazon_product_name
);
if (priceList.length > 0) {
const price = priceList.reduce(minPrice);
console.log("price=", price);
const updatePrice =
"UPDATE products SET yodobashi_product_price = " +
price +
" WHERE id = " +
productArr.id +
" ";
dbClient.query(updatePrice, (err) => {
if (err) {
console.error(err.stack);
}
});
} else {
const soldOut =
"UPDATE products SET yodobashi_product_price = " +
0 +
" WHERE id = " +
productArr.id +
" ";
dbClient.query(soldOut, (err) => {
if (err) {
console.error(err.stack);
}
});
}
} else {
console.log("getPriceはelse判定");
}
productArr = null;
};

const updateDB = async (PRODUCT, BROWSER) => {
const productName = await PRODUCT;
const browser = await BROWSER;
const page = await browser.newPage();
await page.waitFor(100);
for (let i = 0; i < productName.length; i++) {
console.log("for i=", i);
// 下記3文はローカルと同様のため削除する。確認済み
// console.log("productName[i].length=", productName[i].length);
// console.log("productName[i]", productName[i]);
// console.log("productName", productName);
await Promise.all(
productName[i].map(async (productArr) => {
console.log("map i=", i);
console.log("productArr", productArr);
const page = await browser.newPage();
await page.waitFor(100);
await searchProduct(page, productArr.amazon_product_name).catch(() =>
page.close()
);
await getPrice(page, productArr).catch(() => page.close());
await page.close();
productArr = null;
})
).catch(() => {
return null;
});
await productName[i].splice(0);
}
};

const calculateDB = async () => {
const QUERY =
"UPDATE products SET comparison = amazon_product_price - yodobashi_product_price";
await dbClient
.query(QUERY)
.then(() => {
console.log("値段の比較が完了しました");
})
.catch((e) => console.error(e.stack));
};

const cleanUp = async (BROWSER) => {
const browser = await BROWSER;
await Promise.all([
dbClient.on("drain", dbClient.end.bind(dbClient)),
browser.close(),
]);
dbClient.end((err) => {
if (err) {
console.error("DB切断時にエラーが発生しました", err.stack);
process.exit(1);
} else {
console.log("DB切断しました");
process.exit(0);
}
});
};

const notification = async (TEXT, CHANNEL, USERNAME) => {
const notificationText = await TEXT;
const channel = await CHANNEL;
const username = await USERNAME;
await webhook.send({
text: notificationText,
channel: channel,
username: username,
});
};

const main = async () => {
try {
await dbClient.connect();
const loopIndex = await getLoopNum();
const productName = await getProductName(loopIndex);
const browser = await puppeteer.launch({
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"-–disable-dev-shm-usage",
"--disable-gpu",
"--no-first-run",
"--no-zygote",
"--single-process",
],
});
await updateDB(productName, browser);
await calculateDB();
await notification(
"値段比較に成功しました: https://data.heroku.com/dataclips/vfqjszrgbgkstplvsgkotjzpmdzi",
"#ヨドバシ-値段比較",
"ヨドバシ値段比較(成功)"
);
process.on("unhandledRejection", console.dir);
} catch (e) {
await notification(e, "#ヨドバシ-値段比較", "ヨドバシ値段比較(失敗)");
} finally {
const browser = await puppeteer.launch({
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"-–disable-dev-shm-usage",
"--disable-gpu",
"--no-first-run",
"--no-zygote",
"--single-process",
],
});
await cleanUp(browser);
}
};
main();
[cv:issue_marketplace_engineer]