feat: v1
This commit is contained in:
Jonah Fintz
parent
7ba64d9945
commit
1c620b036f
19 changed files with 2745 additions and 91 deletions
1
.dockerignore
Normal file
1
.dockerignore
Normal file
|
|
@ -0,0 +1 @@
|
|||
node_modules/
|
||||
162
.eslintrc.js
Normal file
162
.eslintrc.js
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/*
|
||||
👋 Hi! This file was autogenerated by tslint-to-eslint-config.
|
||||
https://github.com/typescript-eslint/tslint-to-eslint-config
|
||||
|
||||
It represents the closest reasonable ESLint configuration to this
|
||||
project's original TSLint configuration.
|
||||
|
||||
We recommend eventually switching this configuration to extend from
|
||||
the recommended rulesets in typescript-eslint.
|
||||
https://github.com/typescript-eslint/tslint-to-eslint-config/blob/master/docs/FAQs.md
|
||||
|
||||
Happy linting! 💖
|
||||
*/
|
||||
module.exports = {
|
||||
env: {
|
||||
browser: true,
|
||||
es6: true,
|
||||
node: true,
|
||||
},
|
||||
extends: [
|
||||
"plugin:@typescript-eslint/recommended",
|
||||
"plugin:@typescript-eslint/recommended-requiring-type-checking",
|
||||
"prettier",
|
||||
"prettier/@typescript-eslint",
|
||||
],
|
||||
parser: "@typescript-eslint/parser",
|
||||
parserOptions: {
|
||||
project: "tsconfig.json",
|
||||
sourceType: "module",
|
||||
},
|
||||
plugins: [
|
||||
"eslint-plugin-import",
|
||||
"eslint-plugin-jsdoc",
|
||||
"eslint-plugin-prefer-arrow",
|
||||
"@typescript-eslint",
|
||||
],
|
||||
rules: {
|
||||
"@typescript-eslint/adjacent-overload-signatures": "error",
|
||||
"@typescript-eslint/array-type": [
|
||||
"error",
|
||||
{
|
||||
default: "array",
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/ban-types": [
|
||||
"error",
|
||||
{
|
||||
types: {
|
||||
Object: {
|
||||
message: "Avoid using the `Object` type. Did you mean `object`?",
|
||||
},
|
||||
Function: {
|
||||
message:
|
||||
"Avoid using the `Function` type. Prefer a specific function type, like `() => void`.",
|
||||
},
|
||||
Boolean: {
|
||||
message: "Avoid using the `Boolean` type. Did you mean `boolean`?",
|
||||
},
|
||||
Number: {
|
||||
message: "Avoid using the `Number` type. Did you mean `number`?",
|
||||
},
|
||||
String: {
|
||||
message: "Avoid using the `String` type. Did you mean `string`?",
|
||||
},
|
||||
Symbol: {
|
||||
message: "Avoid using the `Symbol` type. Did you mean `symbol`?",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/consistent-type-assertions": "error",
|
||||
"@typescript-eslint/dot-notation": "error",
|
||||
"@typescript-eslint/naming-convention": "error",
|
||||
"@typescript-eslint/no-empty-function": "error",
|
||||
"@typescript-eslint/no-empty-interface": "error",
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-misused-new": "error",
|
||||
"@typescript-eslint/no-namespace": "error",
|
||||
"@typescript-eslint/no-parameter-properties": "off",
|
||||
"@typescript-eslint/no-shadow": [
|
||||
"error",
|
||||
{
|
||||
hoist: "all",
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/no-unused-expressions": "error",
|
||||
"@typescript-eslint/no-use-before-define": "off",
|
||||
"@typescript-eslint/no-var-requires": "error",
|
||||
"@typescript-eslint/prefer-for-of": "error",
|
||||
"@typescript-eslint/prefer-function-type": "error",
|
||||
"@typescript-eslint/prefer-namespace-keyword": "error",
|
||||
"@typescript-eslint/triple-slash-reference": [
|
||||
"error",
|
||||
{
|
||||
path: "always",
|
||||
types: "prefer-import",
|
||||
lib: "always",
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/unified-signatures": "error",
|
||||
"comma-dangle": "off",
|
||||
complexity: "off",
|
||||
"constructor-super": "error",
|
||||
"dot-notation": "error",
|
||||
eqeqeq: ["error", "smart"],
|
||||
"guard-for-in": "error",
|
||||
"id-blacklist": [
|
||||
"error",
|
||||
"any",
|
||||
"Number",
|
||||
"number",
|
||||
"String",
|
||||
"string",
|
||||
"Boolean",
|
||||
"boolean",
|
||||
"Undefined",
|
||||
"undefined",
|
||||
],
|
||||
"id-match": "error",
|
||||
"import/order": "off",
|
||||
"jsdoc/check-alignment": "error",
|
||||
"jsdoc/check-indentation": "error",
|
||||
"jsdoc/newline-after-description": "error",
|
||||
"max-classes-per-file": "off",
|
||||
"new-parens": "error",
|
||||
"no-bitwise": "error",
|
||||
"no-caller": "error",
|
||||
"no-cond-assign": "error",
|
||||
"no-console": "off",
|
||||
"no-debugger": "error",
|
||||
"no-empty": "error",
|
||||
"no-empty-function": "error",
|
||||
"no-eval": "error",
|
||||
"no-fallthrough": "off",
|
||||
"no-invalid-this": "off",
|
||||
"no-new-wrappers": "error",
|
||||
"no-shadow": "error",
|
||||
"no-throw-literal": "error",
|
||||
"no-trailing-spaces": "error",
|
||||
"no-undef-init": "error",
|
||||
"no-underscore-dangle": "error",
|
||||
"no-unsafe-finally": "error",
|
||||
"no-unused-expressions": "error",
|
||||
"no-unused-labels": "error",
|
||||
"no-use-before-define": "off",
|
||||
"no-var": "error",
|
||||
"object-shorthand": "error",
|
||||
"one-var": ["error", "never"],
|
||||
"prefer-arrow/prefer-arrow-functions": "error",
|
||||
"prefer-const": "error",
|
||||
radix: "error",
|
||||
"spaced-comment": [
|
||||
"error",
|
||||
"always",
|
||||
{
|
||||
markers: ["/"],
|
||||
},
|
||||
],
|
||||
"use-isnan": "error",
|
||||
"valid-typeof": "off",
|
||||
},
|
||||
};
|
||||
110
.gitignore
vendored
110
.gitignore
vendored
|
|
@ -2,129 +2,57 @@
|
|||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
# Compiled binary addons (http://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
node_modules
|
||||
jspm_packages
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
# 0x
|
||||
profile-*
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
# mac files
|
||||
.DS_Store
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
# vim swap files
|
||||
*.swp
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
# webstorm
|
||||
.idea
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
out
|
||||
# vscode
|
||||
.vscode
|
||||
*code-workspace
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.*
|
||||
# clinic
|
||||
profile*
|
||||
*clinic*
|
||||
*flamegraph*
|
||||
|
|
|
|||
8
.prettierrc
Executable file
8
.prettierrc
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"printWidth": 100,
|
||||
"semi": true,
|
||||
"singleQuote": false,
|
||||
"trailingComma": "all",
|
||||
"arrowParens": "always",
|
||||
"jsxBracketSameLine": false
|
||||
}
|
||||
27
Dockerfile
Normal file
27
Dockerfile
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
|
||||
FROM node:22.0.0
|
||||
|
||||
WORKDIR /api
|
||||
|
||||
|
||||
## RUN apt-get update && \
|
||||
## apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
|
||||
## libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
|
||||
## libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
|
||||
## libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
|
||||
## libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget unzip graphicsmagick
|
||||
##
|
||||
## ## install chromium
|
||||
## RUN wget -q 'https://playwright.azureedge.net/builds/chromium/1127/chromium-linux-arm64.zip'
|
||||
## RUN unzip chromium-linux-arm64.zip
|
||||
## RUN rm -f ./chromium-linux-arm64.zip
|
||||
|
||||
COPY package.json ./
|
||||
COPY package-lock.json ./
|
||||
|
||||
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD npm run start
|
||||
26
Dockerfile.dev
Normal file
26
Dockerfile.dev
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
FROM node:22.0.0
|
||||
# Create app directory
|
||||
|
||||
WORKDIR /api
|
||||
|
||||
|
||||
## RUN apt-get update && \
|
||||
## apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
|
||||
## libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
|
||||
## libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
|
||||
## libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
|
||||
## libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget unzip graphicsmagick
|
||||
##
|
||||
## ## install chromium
|
||||
## RUN wget -q 'https://playwright.azureedge.net/builds/chromium/1127/chromium-linux-arm64.zip'
|
||||
## RUN unzip chromium-linux-arm64.zip
|
||||
## RUN rm -f ./chromium-linux-arm64.zip
|
||||
|
||||
COPY *.json ./
|
||||
COPY start.js ./
|
||||
|
||||
RUN npm install
|
||||
|
||||
|
||||
## Launch the wait tool and then your application
|
||||
CMD npm run dev
|
||||
16
docker-compose.prod.yaml
Normal file
16
docker-compose.prod.yaml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
services:
|
||||
app:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "3010:3010"
|
||||
volumes:
|
||||
- ./src:/api/src/
|
||||
environment:
|
||||
NODE_ENV: "production"
|
||||
TZ: Europe/Berlin
|
||||
WAIT_HOSTS: mariadb:3306
|
||||
SUPABASE_URL: https://bcgdtkjbsxloiqfhtcfa.supabase.co
|
||||
SUPABASE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImJjZ2R0a2pic3hsb2lxZmh0Y2ZhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3Mjg3NDQ3MTAsImV4cCI6MjA0NDMyMDcxMH0.DxcwwyxU8_dfqBhBHnCYA7Cpciet9XGYg3jxmju9tLQ
|
||||
|
||||
21
docker-compose.yaml
Normal file
21
docker-compose.yaml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
services:
|
||||
app:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.dev
|
||||
ports:
|
||||
- "3010:3010"
|
||||
volumes:
|
||||
- ./src:/api/src/
|
||||
environment:
|
||||
NODE_ENV: "development"
|
||||
TZ: Europe/Berlin
|
||||
WAIT_HOSTS: mariadb:3306
|
||||
MYSQL_HOST: mariadb
|
||||
MYSQL_USER: docker
|
||||
SUPABASE_URL: https://bcgdtkjbsxloiqfhtcfa.supabase.co
|
||||
SUPABASE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImJjZ2R0a2pic3hsb2lxZmh0Y2ZhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3Mjg3NDQ3MTAsImV4cCI6MjA0NDMyMDcxMH0.DxcwwyxU8_dfqBhBHnCYA7Cpciet9XGYg3jxmju9tLQ
|
||||
|
||||
volumes:
|
||||
wool-mysql:
|
||||
|
||||
10
nodemon.json
Executable file
10
nodemon.json
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"watch": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"ext": "ts",
|
||||
"exec": "ts-node --files ./src/app.ts",
|
||||
"args": [
|
||||
"--inspect=5858"
|
||||
]
|
||||
}
|
||||
1798
package-lock.json
generated
Normal file
1798
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
30
package.json
Normal file
30
package.json
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
{
|
||||
"name": "wooldatafetcher",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "ts-node --files ./src/app.ts",
|
||||
"dev": "nodemon"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"@supabase/supabase-js": "^2.45.4",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"axios": "^1.7.3",
|
||||
"dayjs": "^1.11.13",
|
||||
"fastify": "^5.0.0",
|
||||
"jsdom": "^25.0.1",
|
||||
"uuid": "^10.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jsdom": "^21.1.7",
|
||||
"@types/node": "^22.7.5",
|
||||
"nodemon": "^3.1.4",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.6.3"
|
||||
}
|
||||
}
|
||||
3
src/Helpers.ts
Normal file
3
src/Helpers.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export const OnlyKeepDigits = (Input: string): string => {
|
||||
return Input.replace(/\D/g, "");
|
||||
};
|
||||
352
src/LanaGrossaIntegration.ts
Normal file
352
src/LanaGrossaIntegration.ts
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
import axios from "axios";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { OnlyKeepDigits } from "./Helpers";
|
||||
import { SupabaseInstance } from "./Supabase";
|
||||
import dayjs from "dayjs";
|
||||
|
||||
export type TWoolFetcherStats = {
|
||||
FoundLinks: number;
|
||||
Date: string;
|
||||
UpdatedWools: number;
|
||||
NewWools: number;
|
||||
UpdatedVariants: number;
|
||||
NewVariants: number;
|
||||
Errors: number;
|
||||
ErrorLinks: string[];
|
||||
ErrorVariants: number;
|
||||
};
|
||||
|
||||
export default class LanaGrossaIntegration {
|
||||
public CurrentStats: TWoolFetcherStats = {
|
||||
FoundLinks: 0,
|
||||
Date: dayjs().format("YYYY-MM-DD HH:mm:ss"),
|
||||
Errors: 0,
|
||||
ErrorLinks: [],
|
||||
ErrorVariants: 0,
|
||||
UpdatedWools: 0,
|
||||
NewWools: 0,
|
||||
UpdatedVariants: 0,
|
||||
NewVariants: 0,
|
||||
};
|
||||
|
||||
public StatList: TWoolFetcherStats[] = [];
|
||||
|
||||
public LinkQueue: string[] = [];
|
||||
|
||||
private async FindLinks(): Promise<string[]> {
|
||||
const Links: string[] = [];
|
||||
|
||||
let HasMore = true;
|
||||
let Page = 1;
|
||||
|
||||
while (HasMore) {
|
||||
const Params: any = {
|
||||
action: "products-overview",
|
||||
locale: "de_DE",
|
||||
};
|
||||
|
||||
if (Page > 1) {
|
||||
Params.pg = Page;
|
||||
}
|
||||
|
||||
let ParamsParts = [];
|
||||
for (const Key of Object.keys(Params)) {
|
||||
ParamsParts.push(`${Key}=${Params[Key]}`);
|
||||
}
|
||||
|
||||
const Response = await axios.get("https://www.lana-grossa.de/wp/wp-admin/admin-ajax.php", {
|
||||
params: Params,
|
||||
});
|
||||
|
||||
if (Response.data.items === "") {
|
||||
HasMore = false;
|
||||
} else {
|
||||
Page += 1;
|
||||
|
||||
const LinkBase = "https://www.lana-grossa.de/garne/detail/";
|
||||
|
||||
const HTML: string = Response.data.items;
|
||||
|
||||
const Matches = HTML.match(/<a href="([^"]+)"/g);
|
||||
|
||||
if (Matches) {
|
||||
for (const Match of Matches) {
|
||||
const Link = Match.replace('<a href="', "").replace('"', "");
|
||||
|
||||
if (Link.startsWith(LinkBase)) {
|
||||
Links.push(Link);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Links;
|
||||
}
|
||||
|
||||
public async RunFetcher(): Promise<void> {
|
||||
// refetch links if empty
|
||||
if (this.LinkQueue.length === 0) {
|
||||
this.LinkQueue = await this.FindLinks();
|
||||
this.CurrentStats = {
|
||||
FoundLinks: this.LinkQueue.length,
|
||||
Date: dayjs().format("YYYY-MM-DD HH:mm:ss"),
|
||||
Errors: 0,
|
||||
ErrorLinks: [],
|
||||
ErrorVariants: 0,
|
||||
UpdatedWools: 0,
|
||||
NewWools: 0,
|
||||
UpdatedVariants: 0,
|
||||
NewVariants: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// run through 10 links
|
||||
const LinksToProcess = this.LinkQueue.splice(0, 10);
|
||||
|
||||
for (const Link of LinksToProcess) {
|
||||
const Stats = await this.ExtractDataFromPage(Link);
|
||||
|
||||
if (Stats.IsError) {
|
||||
this.CurrentStats.Errors += 1;
|
||||
this.CurrentStats.ErrorLinks.push(Link);
|
||||
}
|
||||
if (Stats.IsNew) {
|
||||
this.CurrentStats.NewWools += 1;
|
||||
} else {
|
||||
this.CurrentStats.UpdatedWools += 1;
|
||||
}
|
||||
|
||||
if (Stats.IsVariantError) {
|
||||
this.CurrentStats.ErrorVariants += 1;
|
||||
}
|
||||
|
||||
this.CurrentStats.UpdatedVariants += Stats.VariantsUpdated;
|
||||
this.CurrentStats.NewVariants += Stats.VariantsNew;
|
||||
}
|
||||
|
||||
if (this.LinkQueue.length === 0) {
|
||||
// add to top of list
|
||||
this.StatList.unshift(this.CurrentStats);
|
||||
|
||||
// only keep 10 stats in statlist
|
||||
if (this.StatList.length > 10) {
|
||||
this.StatList.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ExtractDataFromPage(Link: string): Promise<{
|
||||
IsNew: boolean;
|
||||
IsVariantError: boolean;
|
||||
IsError: boolean;
|
||||
VariantsUpdated: number;
|
||||
VariantsNew: number;
|
||||
}> {
|
||||
const Stats = {
|
||||
IsNew: false,
|
||||
IsError: false,
|
||||
IsVariantError: false,
|
||||
VariantsUpdated: 0,
|
||||
VariantsNew: 0,
|
||||
};
|
||||
const PageResponse = await axios.get(Link);
|
||||
|
||||
if (PageResponse.status !== 200) {
|
||||
Stats.IsError = true;
|
||||
return Stats;
|
||||
}
|
||||
|
||||
const HTML: string = PageResponse.data;
|
||||
|
||||
const Dom = new JSDOM(HTML);
|
||||
const Document = Dom.window.document;
|
||||
|
||||
// get name
|
||||
const Name = Document.querySelector(".page-headline .text-title1").textContent;
|
||||
|
||||
// get weight
|
||||
const WeightText = Document.querySelector(".details-list .weight").textContent;
|
||||
let Weight: number = 0;
|
||||
if (WeightText.toLocaleLowerCase().indexOf("kg") > 0) {
|
||||
Weight = parseFloat(WeightText.replace("kg", "").trim()) * 1000;
|
||||
} else if (WeightText.toLocaleLowerCase().indexOf("g") > 0) {
|
||||
Weight = parseFloat(WeightText.replace("g", "").trim());
|
||||
}
|
||||
|
||||
// get run length
|
||||
const RunLengthText = Document.querySelector(".details-list .length").textContent;
|
||||
let RunLength: number = 0;
|
||||
if (RunLengthText.toLocaleLowerCase().indexOf("m") > 0) {
|
||||
RunLength = parseFloat(RunLengthText.replace("m", "").trim());
|
||||
}
|
||||
|
||||
// get needle size
|
||||
let NeedleSizeMin: number | null = null;
|
||||
let NeedleSizeMax: number | null = null;
|
||||
|
||||
if (Document.querySelector(".details-list .needle-size") != null) {
|
||||
const NeedleSizeText = Document.querySelector(".details-list .needle-size").textContent;
|
||||
|
||||
if (NeedleSizeText.indexOf("-") > 0) {
|
||||
const Parts = NeedleSizeText.split("-");
|
||||
if (Parts.length === 2) {
|
||||
NeedleSizeMin = parseFloat(Parts[0].replace(",", "."));
|
||||
NeedleSizeMax = parseFloat(Parts[1].replace(",", "."));
|
||||
}
|
||||
} else {
|
||||
NeedleSizeMin = parseFloat(NeedleSizeText.replace(",", "."));
|
||||
NeedleSizeMax = parseFloat(NeedleSizeText.replace(",", "."));
|
||||
}
|
||||
}
|
||||
|
||||
// get stitch test
|
||||
let StitchTestR: number | null = 0;
|
||||
let StitchTestM: number | null = 0;
|
||||
|
||||
if (Document.querySelector(".details-list .mesh-probe") != null) {
|
||||
const StitchTestText = Document.querySelector(".details-list .mesh-probe")?.textContent;
|
||||
const StichTestParts = StitchTestText.split(",");
|
||||
if (StichTestParts.length === 2) {
|
||||
StitchTestR = parseFloat(OnlyKeepDigits(StichTestParts[0]));
|
||||
StitchTestM = parseFloat(OnlyKeepDigits(StichTestParts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
// get composition
|
||||
const CompositionElements = Document.querySelectorAll(".module-material-details > .lc > p");
|
||||
const Composition: string[] = [];
|
||||
for (const Element of CompositionElements) {
|
||||
Composition.push(Element.textContent);
|
||||
}
|
||||
|
||||
const Key = "LanaGrossa_" + Name + "_" + Weight;
|
||||
|
||||
// check if already exists
|
||||
const Existing = await SupabaseInstance.from("Wool")
|
||||
.select("id, updated_at", { count: "exact" })
|
||||
.eq("key", Key);
|
||||
|
||||
let UUID = Existing.data[0]?.id;
|
||||
|
||||
if (Existing.count == null || Existing.count == 0) {
|
||||
Stats.IsNew = true;
|
||||
const Result = await SupabaseInstance.from("Wool")
|
||||
.insert({
|
||||
name: Name,
|
||||
key: Key,
|
||||
weight: Weight,
|
||||
maker: "LanaGrossa",
|
||||
run_length: RunLength,
|
||||
composition: Composition,
|
||||
needle_size_max: NeedleSizeMax,
|
||||
needle_size_min: NeedleSizeMin,
|
||||
stitch_test_m: StitchTestM,
|
||||
stitch_test_r: StitchTestR,
|
||||
})
|
||||
.select("id");
|
||||
|
||||
UUID = Result.data[0].id;
|
||||
} else {
|
||||
// only if updated_at is older than 1 day
|
||||
Stats.IsNew = false;
|
||||
|
||||
const UpdatedAt = Existing.data[0].updated_at;
|
||||
if (dayjs().diff(dayjs(UpdatedAt), "day") < 1) {
|
||||
console.log(
|
||||
"Skipping",
|
||||
Name,
|
||||
"as it was updated less than a day ago, also skipping variants",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// update
|
||||
await SupabaseInstance.from("Wool")
|
||||
.update({
|
||||
maker: "LanaGrossa",
|
||||
run_length: RunLength,
|
||||
composition: Composition,
|
||||
needle_size_max: NeedleSizeMax,
|
||||
needle_size_min: NeedleSizeMin,
|
||||
stitch_test_m: StitchTestM,
|
||||
stitch_test_r: StitchTestR,
|
||||
updated_at: dayjs().toISOString(),
|
||||
})
|
||||
.eq("key", Key);
|
||||
}
|
||||
|
||||
// extract variants
|
||||
const JavaScripts = Document.querySelectorAll("body > script");
|
||||
|
||||
let EanScript = null;
|
||||
|
||||
for (const ScriptObject of JavaScripts) {
|
||||
const Script = ScriptObject.textContent;
|
||||
|
||||
if (Script.indexOf("eans") > -1) {
|
||||
EanScript = Script;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (EanScript != null) {
|
||||
Dom.window.eval(EanScript.replaceAll("var ", ""));
|
||||
|
||||
const EANs: string[] = Dom.window.eval("eans") as any;
|
||||
const ColorNames: string[] = Dom.window.eval("colorNames") as any;
|
||||
const ColorCodes: string[] = Dom.window.eval("colorCodes") as any;
|
||||
|
||||
if (EANs.length !== ColorNames.length || EANs.length !== ColorCodes.length) {
|
||||
console.error("Lengths do not match");
|
||||
}
|
||||
|
||||
for (let Index = 0; Index < EANs.length; Index++) {
|
||||
const EAN = EANs[Index];
|
||||
const ColorName = ColorNames[Index];
|
||||
const ColorCode = ColorCodes[Index];
|
||||
|
||||
const VariantKey = "LanaGrossa_" + Name + "_" + EAN;
|
||||
|
||||
const ExistingVariant = await SupabaseInstance.from("WoolVariants")
|
||||
.select("id, updated_at", { count: "exact" })
|
||||
.eq("key", VariantKey);
|
||||
|
||||
if (ExistingVariant.count == null || ExistingVariant.count == 0) {
|
||||
Stats.VariantsNew += 1;
|
||||
await SupabaseInstance.from("WoolVariants").insert({
|
||||
key: VariantKey,
|
||||
wool_id: UUID,
|
||||
ean: EAN,
|
||||
color_name: ColorName,
|
||||
color_code: ColorCode,
|
||||
});
|
||||
} else {
|
||||
Stats.VariantsUpdated += 1;
|
||||
// only if updated_at is older than 1 day
|
||||
const UpdatedAt = Existing.data[0].updated_at;
|
||||
if (dayjs().diff(dayjs(UpdatedAt), "day") < 1) {
|
||||
console.log("Skipping Variant", EAN, "as it was updated less than a day ago");
|
||||
continue;
|
||||
}
|
||||
|
||||
// update
|
||||
await SupabaseInstance.from("WoolVariants")
|
||||
.update({
|
||||
wool_id: UUID,
|
||||
color: ColorName,
|
||||
color_name: ColorName,
|
||||
color_code: ColorCode,
|
||||
updated_at: dayjs().toISOString(),
|
||||
})
|
||||
.eq("key", VariantKey);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Stats.IsVariantError = true;
|
||||
}
|
||||
|
||||
return Stats;
|
||||
}
|
||||
}
|
||||
7
src/Supabase.ts
Normal file
7
src/Supabase.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { createClient } from "@supabase/supabase-js";
|
||||
import { Database } from "supabase";
|
||||
|
||||
export const SupabaseInstance = createClient<Database>(
|
||||
process.env.SUPABASE_URL,
|
||||
process.env.SUPABASE_KEY,
|
||||
);
|
||||
33
src/app.ts
Normal file
33
src/app.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import fastify from "fastify";
|
||||
import LanaGrossaIntegration from "./LanaGrossaIntegration";
|
||||
|
||||
const server = fastify();
|
||||
|
||||
const LanaGrossaIntegrationInstance = new LanaGrossaIntegration();
|
||||
|
||||
// start update every 24 hours
|
||||
setInterval(() => {
|
||||
LanaGrossaIntegrationInstance.RunFetcher();
|
||||
}, 1000 * 60 * 60 * 1);
|
||||
LanaGrossaIntegrationInstance.RunFetcher();
|
||||
|
||||
server.get("/stats", async (request, reply) => {
|
||||
const CurrentStats = LanaGrossaIntegrationInstance.CurrentStats;
|
||||
const StatList = LanaGrossaIntegrationInstance.StatList;
|
||||
|
||||
const RemainingLinks = LanaGrossaIntegrationInstance.LinkQueue.length;
|
||||
|
||||
return {
|
||||
RemainingLinks,
|
||||
CurrentStats,
|
||||
StatList,
|
||||
};
|
||||
});
|
||||
|
||||
server.listen({ port: 3010, host: "0.0.0.0" }, (err, address) => {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`Server listening at ${address}`);
|
||||
});
|
||||
191
src/types/supabase.ts
Normal file
191
src/types/supabase.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
export type Json =
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| null
|
||||
| { [key: string]: Json | undefined }
|
||||
| Json[]
|
||||
|
||||
export type Database = {
|
||||
public: {
|
||||
Tables: {
|
||||
Wool: {
|
||||
Row: {
|
||||
composition: string[] | null
|
||||
created_at: string
|
||||
id: string
|
||||
key: string
|
||||
maker: Database["public"]["Enums"]["WoolMaker"] | null
|
||||
name: string
|
||||
needle_size_max: number | null
|
||||
needle_size_min: number | null
|
||||
run_length: number | null
|
||||
stitch_test_m: number | null
|
||||
stitch_test_r: number | null
|
||||
updated_at: string | null
|
||||
weight: number | null
|
||||
}
|
||||
Insert: {
|
||||
composition?: string[] | null
|
||||
created_at?: string
|
||||
id?: string
|
||||
key: string
|
||||
maker?: Database["public"]["Enums"]["WoolMaker"] | null
|
||||
name?: string
|
||||
needle_size_max?: number | null
|
||||
needle_size_min?: number | null
|
||||
run_length?: number | null
|
||||
stitch_test_m?: number | null
|
||||
stitch_test_r?: number | null
|
||||
updated_at?: string | null
|
||||
weight?: number | null
|
||||
}
|
||||
Update: {
|
||||
composition?: string[] | null
|
||||
created_at?: string
|
||||
id?: string
|
||||
key?: string
|
||||
maker?: Database["public"]["Enums"]["WoolMaker"] | null
|
||||
name?: string
|
||||
needle_size_max?: number | null
|
||||
needle_size_min?: number | null
|
||||
run_length?: number | null
|
||||
stitch_test_m?: number | null
|
||||
stitch_test_r?: number | null
|
||||
updated_at?: string | null
|
||||
weight?: number | null
|
||||
}
|
||||
Relationships: []
|
||||
}
|
||||
WoolVariants: {
|
||||
Row: {
|
||||
color: string | null
|
||||
created_at: string
|
||||
ean: string | null
|
||||
id: string
|
||||
key: string | null
|
||||
wool_id: string | null
|
||||
}
|
||||
Insert: {
|
||||
color?: string | null
|
||||
created_at?: string
|
||||
ean?: string | null
|
||||
id?: string
|
||||
key?: string | null
|
||||
wool_id?: string | null
|
||||
}
|
||||
Update: {
|
||||
color?: string | null
|
||||
created_at?: string
|
||||
ean?: string | null
|
||||
id?: string
|
||||
key?: string | null
|
||||
wool_id?: string | null
|
||||
}
|
||||
Relationships: [
|
||||
{
|
||||
foreignKeyName: "WoolVariants_wool_id_fkey"
|
||||
columns: ["wool_id"]
|
||||
isOneToOne: false
|
||||
referencedRelation: "Wool"
|
||||
referencedColumns: ["id"]
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
Views: {
|
||||
[_ in never]: never
|
||||
}
|
||||
Functions: {
|
||||
[_ in never]: never
|
||||
}
|
||||
Enums: {
|
||||
WoolMaker: "LanaGrossa"
|
||||
}
|
||||
CompositeTypes: {
|
||||
[_ in never]: never
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type PublicSchema = Database[Extract<keyof Database, "public">]
|
||||
|
||||
export type Tables<
|
||||
PublicTableNameOrOptions extends
|
||||
| keyof (PublicSchema["Tables"] & PublicSchema["Views"])
|
||||
| { schema: keyof Database },
|
||||
TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? keyof (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
|
||||
Database[PublicTableNameOrOptions["schema"]]["Views"])
|
||||
: never = never,
|
||||
> = PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
|
||||
Database[PublicTableNameOrOptions["schema"]]["Views"])[TableName] extends {
|
||||
Row: infer R
|
||||
}
|
||||
? R
|
||||
: never
|
||||
: PublicTableNameOrOptions extends keyof (PublicSchema["Tables"] &
|
||||
PublicSchema["Views"])
|
||||
? (PublicSchema["Tables"] &
|
||||
PublicSchema["Views"])[PublicTableNameOrOptions] extends {
|
||||
Row: infer R
|
||||
}
|
||||
? R
|
||||
: never
|
||||
: never
|
||||
|
||||
export type TablesInsert<
|
||||
PublicTableNameOrOptions extends
|
||||
| keyof PublicSchema["Tables"]
|
||||
| { schema: keyof Database },
|
||||
TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
|
||||
: never = never,
|
||||
> = PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
|
||||
Insert: infer I
|
||||
}
|
||||
? I
|
||||
: never
|
||||
: PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
|
||||
? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
|
||||
Insert: infer I
|
||||
}
|
||||
? I
|
||||
: never
|
||||
: never
|
||||
|
||||
export type TablesUpdate<
|
||||
PublicTableNameOrOptions extends
|
||||
| keyof PublicSchema["Tables"]
|
||||
| { schema: keyof Database },
|
||||
TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
|
||||
: never = never,
|
||||
> = PublicTableNameOrOptions extends { schema: keyof Database }
|
||||
? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
|
||||
Update: infer U
|
||||
}
|
||||
? U
|
||||
: never
|
||||
: PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
|
||||
? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
|
||||
Update: infer U
|
||||
}
|
||||
? U
|
||||
: never
|
||||
: never
|
||||
|
||||
export type Enums<
|
||||
PublicEnumNameOrOptions extends
|
||||
| keyof PublicSchema["Enums"]
|
||||
| { schema: keyof Database },
|
||||
EnumName extends PublicEnumNameOrOptions extends { schema: keyof Database }
|
||||
? keyof Database[PublicEnumNameOrOptions["schema"]]["Enums"]
|
||||
: never = never,
|
||||
> = PublicEnumNameOrOptions extends { schema: keyof Database }
|
||||
? Database[PublicEnumNameOrOptions["schema"]]["Enums"][EnumName]
|
||||
: PublicEnumNameOrOptions extends keyof PublicSchema["Enums"]
|
||||
? PublicSchema["Enums"][PublicEnumNameOrOptions]
|
||||
: never
|
||||
14
start.js
Executable file
14
start.js
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
// Transpile all code following this line with babel and use '@babel/preset-env' (aka ES6) preset.
|
||||
require("@babel/register")({
|
||||
presets: [
|
||||
[
|
||||
"@babel/preset-env",
|
||||
{
|
||||
targets: { browsers: ["last 2 chrome versions"] },
|
||||
},
|
||||
],
|
||||
],
|
||||
});
|
||||
|
||||
// Import the rest of our application.
|
||||
module.exports = require("./src/app.ts");
|
||||
5
supabase-generate-types.sh
Executable file
5
supabase-generate-types.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
## Set access Token
|
||||
export SUPABASE_ACCESS_TOKEN=sbp_9a95062bb5df26f382ea79d0d43c6570f4071af8
|
||||
|
||||
## Generate types
|
||||
npx --yes supabase gen types typescript --project-id "bcgdtkjbsxloiqfhtcfa" --schema public >src/types/supabase.ts
|
||||
22
tsconfig.json
Normal file
22
tsconfig.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "commonjs",
|
||||
// "lib": [],
|
||||
"sourceMap": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"moduleResolution": "node",
|
||||
"resolveJsonModule": true,
|
||||
"esModuleInterop": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"*": ["node_modules/*", "src/types/*"]
|
||||
},
|
||||
"typeRoots": ["src/types/"]
|
||||
// "rootDirs": [],
|
||||
// "typeRoots": [],
|
||||
// "types": [],
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue