decode HTML entities in scrape.js

This commit is contained in:
Clarence Chan committed 2023-09-13 11:17:30 -04:00
1 parent 20657984a2
commit cdb70eed9b
3 files changed
+22 -4

No files matched your search

+17
View File
@@ -9,6 +9,7 @@
"version": "1.0.0", "version": "1.0.0",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"html-entities": "^2.4.0",
"node-html-parser": "^6.1.9" "node-html-parser": "^6.1.9"
} }
}, },
@@ -122,6 +123,22 @@
"he": "bin/he" "he": "bin/he"
} }
}, },
"node_modules/html-entities": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.4.0.tgz",
"integrity": "sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==",
"dev": true,
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/mdevils"
},
{
"type": "patreon",
"url": "https://patreon.com/mdevils"
}
]
},
"node_modules/node-html-parser": { "node_modules/node-html-parser": {
"version": "6.1.9", "version": "6.1.9",
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.9.tgz", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.9.tgz",
+1
View File
@@ -9,6 +9,7 @@
"author": "Clarence Chan", "author": "Clarence Chan",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"html-entities": "^2.4.0",
"node-html-parser": "^6.1.9" "node-html-parser": "^6.1.9"
}, },
"type": "module" "type": "module"
+4 -4
View File
@@ -1,4 +1,5 @@
import { parse } from "node-html-parser"; import { parse } from "node-html-parser";
import { decode } from "html-entities";
async function searchEvents() { async function searchEvents() {
let url = `https://www.tapology.com/search?term=ufc&commit=Submit&model%5Bevents%5D=eventsSearch`; let url = `https://www.tapology.com/search?term=ufc&commit=Submit&model%5Bevents%5D=eventsSearch`;
@@ -37,7 +38,7 @@ async function getDetails(event) {
return date; return date;
} }
const { name, link: url } = event; let { name, link: url } = event;
try { try {
const response = await fetch(url); const response = await fetch(url);
const text = await response.text(); const text = await response.text();
@@ -63,9 +64,11 @@ async function getDetails(event) {
let innerText = li.innerText.split("\n"); let innerText = li.innerText.split("\n");
let fight = let fight =
innerText[2] + " vs. " + innerText[4] + " (" + innerText[7] + ")"; innerText[2] + " vs. " + innerText[4] + " (" + innerText[7] + ")";
fight = decode(fight);
fights.push(fight); fights.push(fight);
} }
[name, venue, location] = [decode(name), decode(venue), decode(location)];
return { name, link: url, date, venue, location, fights }; return { name, link: url, date, venue, location, fights };
} catch (error) { } catch (error) {
console.error(error); console.error(error);
@@ -74,7 +77,6 @@ async function getDetails(event) {
async function getDetailedEvents() { async function getDetailedEvents() {
let events = []; let events = [];
try { try {
events = await searchEvents(); events = await searchEvents();
events = events.map((event) => getDetails(event)); events = events.map((event) => getDetails(event));
@@ -84,5 +86,3 @@ async function getDetailedEvents() {
console.error(error); console.error(error);
} }
} }
getDetailedEvents();