feat: v1

src/Helpers.ts

@@ -0,0 +1,3 @@
+export const OnlyKeepDigits = (Input: string): string => {
+  return Input.replace(/\D/g, "");
+};

src/LanaGrossaIntegration.ts

@@ -0,0 +1,352 @@
+import axios from "axios";
+import { JSDOM } from "jsdom";
+import { OnlyKeepDigits } from "./Helpers";
+import { SupabaseInstance } from "./Supabase";
+import dayjs from "dayjs";
+
+export type TWoolFetcherStats = {
+  FoundLinks: number;
+  Date: string;
+  UpdatedWools: number;
+  NewWools: number;
+  UpdatedVariants: number;
+  NewVariants: number;
+  Errors: number;
+  ErrorLinks: string[];
+  ErrorVariants: number;
+};
+
+export default class LanaGrossaIntegration {
+  public CurrentStats: TWoolFetcherStats = {
+    FoundLinks: 0,
+    Date: dayjs().format("YYYY-MM-DD HH:mm:ss"),
+    Errors: 0,
+    ErrorLinks: [],
+    ErrorVariants: 0,
+    UpdatedWools: 0,
+    NewWools: 0,
+    UpdatedVariants: 0,
+    NewVariants: 0,
+  };
+
+  public StatList: TWoolFetcherStats[] = [];
+
+  public LinkQueue: string[] = [];
+
+  private async FindLinks(): Promise<string[]> {
+    const Links: string[] = [];
+
+    let HasMore = true;
+    let Page = 1;
+
+    while (HasMore) {
+      const Params: any = {
+        action: "products-overview",
+        locale: "de_DE",
+      };
+
+      if (Page > 1) {
+        Params.pg = Page;
+      }
+
+      let ParamsParts = [];
+      for (const Key of Object.keys(Params)) {
+        ParamsParts.push(`${Key}=${Params[Key]}`);
+      }
+
+      const Response = await axios.get("https://www.lana-grossa.de/wp/wp-admin/admin-ajax.php", {
+        params: Params,
+      });
+
+      if (Response.data.items === "") {
+        HasMore = false;
+      } else {
+        Page += 1;
+
+        const LinkBase = "https://www.lana-grossa.de/garne/detail/";
+
+        const HTML: string = Response.data.items;
+
+        const Matches = HTML.match(/<a href="([^"]+)"/g);
+
+        if (Matches) {
+          for (const Match of Matches) {
+            const Link = Match.replace('<a href="', "").replace('"', "");
+
+            if (Link.startsWith(LinkBase)) {
+              Links.push(Link);
+            }
+          }
+        }
+      }
+    }
+
+    return Links;
+  }
+
+  public async RunFetcher(): Promise<void> {
+    // refetch links if empty
+    if (this.LinkQueue.length === 0) {
+      this.LinkQueue = await this.FindLinks();
+      this.CurrentStats = {
+        FoundLinks: this.LinkQueue.length,
+        Date: dayjs().format("YYYY-MM-DD HH:mm:ss"),
+        Errors: 0,
+        ErrorLinks: [],
+        ErrorVariants: 0,
+        UpdatedWools: 0,
+        NewWools: 0,
+        UpdatedVariants: 0,
+        NewVariants: 0,
+      };
+    }
+
+    // run through 10 links
+    const LinksToProcess = this.LinkQueue.splice(0, 10);
+
+    for (const Link of LinksToProcess) {
+      const Stats = await this.ExtractDataFromPage(Link);
+
+      if (Stats.IsError) {
+        this.CurrentStats.Errors += 1;
+        this.CurrentStats.ErrorLinks.push(Link);
+      }
+      if (Stats.IsNew) {
+        this.CurrentStats.NewWools += 1;
+      } else {
+        this.CurrentStats.UpdatedWools += 1;
+      }
+
+      if (Stats.IsVariantError) {
+        this.CurrentStats.ErrorVariants += 1;
+      }
+
+      this.CurrentStats.UpdatedVariants += Stats.VariantsUpdated;
+      this.CurrentStats.NewVariants += Stats.VariantsNew;
+    }
+
+    if (this.LinkQueue.length === 0) {
+      // add to top of list
+      this.StatList.unshift(this.CurrentStats);
+
+      // only keep 10 stats in statlist
+      if (this.StatList.length > 10) {
+        this.StatList.pop();
+      }
+    }
+  }
+
+  private async ExtractDataFromPage(Link: string): Promise<{
+    IsNew: boolean;
+    IsVariantError: boolean;
+    IsError: boolean;
+    VariantsUpdated: number;
+    VariantsNew: number;
+  }> {
+    const Stats = {
+      IsNew: false,
+      IsError: false,
+      IsVariantError: false,
+      VariantsUpdated: 0,
+      VariantsNew: 0,
+    };
+    const PageResponse = await axios.get(Link);
+
+    if (PageResponse.status !== 200) {
+      Stats.IsError = true;
+      return Stats;
+    }
+
+    const HTML: string = PageResponse.data;
+
+    const Dom = new JSDOM(HTML);
+    const Document = Dom.window.document;
+
+    // get name
+    const Name = Document.querySelector(".page-headline .text-title1").textContent;
+
+    // get weight
+    const WeightText = Document.querySelector(".details-list .weight").textContent;
+    let Weight: number = 0;
+    if (WeightText.toLocaleLowerCase().indexOf("kg") > 0) {
+      Weight = parseFloat(WeightText.replace("kg", "").trim()) * 1000;
+    } else if (WeightText.toLocaleLowerCase().indexOf("g") > 0) {
+      Weight = parseFloat(WeightText.replace("g", "").trim());
+    }
+
+    // get run length
+    const RunLengthText = Document.querySelector(".details-list .length").textContent;
+    let RunLength: number = 0;
+    if (RunLengthText.toLocaleLowerCase().indexOf("m") > 0) {
+      RunLength = parseFloat(RunLengthText.replace("m", "").trim());
+    }
+
+    // get needle size
+    let NeedleSizeMin: number | null = null;
+    let NeedleSizeMax: number | null = null;
+
+    if (Document.querySelector(".details-list .needle-size") != null) {
+      const NeedleSizeText = Document.querySelector(".details-list .needle-size").textContent;
+
+      if (NeedleSizeText.indexOf("-") > 0) {
+        const Parts = NeedleSizeText.split("-");
+        if (Parts.length === 2) {
+          NeedleSizeMin = parseFloat(Parts[0].replace(",", "."));
+          NeedleSizeMax = parseFloat(Parts[1].replace(",", "."));
+        }
+      } else {
+        NeedleSizeMin = parseFloat(NeedleSizeText.replace(",", "."));
+        NeedleSizeMax = parseFloat(NeedleSizeText.replace(",", "."));
+      }
+    }
+
+    // get stitch test
+    let StitchTestR: number | null = 0;
+    let StitchTestM: number | null = 0;
+
+    if (Document.querySelector(".details-list .mesh-probe") != null) {
+      const StitchTestText = Document.querySelector(".details-list .mesh-probe")?.textContent;
+      const StichTestParts = StitchTestText.split(",");
+      if (StichTestParts.length === 2) {
+        StitchTestR = parseFloat(OnlyKeepDigits(StichTestParts[0]));
+        StitchTestM = parseFloat(OnlyKeepDigits(StichTestParts[1]));
+      }
+    }
+
+    // get composition
+    const CompositionElements = Document.querySelectorAll(".module-material-details > .lc > p");
+    const Composition: string[] = [];
+    for (const Element of CompositionElements) {
+      Composition.push(Element.textContent);
+    }
+
+    const Key = "LanaGrossa_" + Name + "_" + Weight;
+
+    // check if already exists
+    const Existing = await SupabaseInstance.from("Wool")
+      .select("id, updated_at", { count: "exact" })
+      .eq("key", Key);
+
+    let UUID = Existing.data[0]?.id;
+
+    if (Existing.count == null || Existing.count == 0) {
+      Stats.IsNew = true;
+      const Result = await SupabaseInstance.from("Wool")
+        .insert({
+          name: Name,
+          key: Key,
+          weight: Weight,
+          maker: "LanaGrossa",
+          run_length: RunLength,
+          composition: Composition,
+          needle_size_max: NeedleSizeMax,
+          needle_size_min: NeedleSizeMin,
+          stitch_test_m: StitchTestM,
+          stitch_test_r: StitchTestR,
+        })
+        .select("id");
+
+      UUID = Result.data[0].id;
+    } else {
+      // only if updated_at is older than 1 day
+      Stats.IsNew = false;
+
+      const UpdatedAt = Existing.data[0].updated_at;
+      if (dayjs().diff(dayjs(UpdatedAt), "day") < 1) {
+        console.log(
+          "Skipping",
+          Name,
+          "as it was updated less than a day ago, also skipping variants",
+        );
+        return;
+      }
+
+      // update
+      await SupabaseInstance.from("Wool")
+        .update({
+          maker: "LanaGrossa",
+          run_length: RunLength,
+          composition: Composition,
+          needle_size_max: NeedleSizeMax,
+          needle_size_min: NeedleSizeMin,
+          stitch_test_m: StitchTestM,
+          stitch_test_r: StitchTestR,
+          updated_at: dayjs().toISOString(),
+        })
+        .eq("key", Key);
+    }
+
+    // extract variants
+    const JavaScripts = Document.querySelectorAll("body > script");
+
+    let EanScript = null;
+
+    for (const ScriptObject of JavaScripts) {
+      const Script = ScriptObject.textContent;
+
+      if (Script.indexOf("eans") > -1) {
+        EanScript = Script;
+
+        break;
+      }
+    }
+
+    if (EanScript != null) {
+      Dom.window.eval(EanScript.replaceAll("var ", ""));
+
+      const EANs: string[] = Dom.window.eval("eans") as any;
+      const ColorNames: string[] = Dom.window.eval("colorNames") as any;
+      const ColorCodes: string[] = Dom.window.eval("colorCodes") as any;
+
+      if (EANs.length !== ColorNames.length || EANs.length !== ColorCodes.length) {
+        console.error("Lengths do not match");
+      }
+
+      for (let Index = 0; Index < EANs.length; Index++) {
+        const EAN = EANs[Index];
+        const ColorName = ColorNames[Index];
+        const ColorCode = ColorCodes[Index];
+
+        const VariantKey = "LanaGrossa_" + Name + "_" + EAN;
+
+        const ExistingVariant = await SupabaseInstance.from("WoolVariants")
+          .select("id, updated_at", { count: "exact" })
+          .eq("key", VariantKey);
+
+        if (ExistingVariant.count == null || ExistingVariant.count == 0) {
+          Stats.VariantsNew += 1;
+          await SupabaseInstance.from("WoolVariants").insert({
+            key: VariantKey,
+            wool_id: UUID,
+            ean: EAN,
+            color_name: ColorName,
+            color_code: ColorCode,
+          });
+        } else {
+          Stats.VariantsUpdated += 1;
+          // only if updated_at is older than 1 day
+          const UpdatedAt = Existing.data[0].updated_at;
+          if (dayjs().diff(dayjs(UpdatedAt), "day") < 1) {
+            console.log("Skipping Variant", EAN, "as it was updated less than a day ago");
+            continue;
+          }
+
+          // update
+          await SupabaseInstance.from("WoolVariants")
+            .update({
+              wool_id: UUID,
+              color: ColorName,
+              color_name: ColorName,
+              color_code: ColorCode,
+              updated_at: dayjs().toISOString(),
+            })
+            .eq("key", VariantKey);
+        }
+      }
+    } else {
+      Stats.IsVariantError = true;
+    }
+
+    return Stats;
+  }
+}

src/Supabase.ts

@@ -0,0 +1,7 @@
+import { createClient } from "@supabase/supabase-js";
+import { Database } from "supabase";
+
+export const SupabaseInstance = createClient<Database>(
+  process.env.SUPABASE_URL,
+  process.env.SUPABASE_KEY,
+);

src/app.ts

@@ -0,0 +1,33 @@
+import fastify from "fastify";
+import LanaGrossaIntegration from "./LanaGrossaIntegration";
+
+const server = fastify();
+
+const LanaGrossaIntegrationInstance = new LanaGrossaIntegration();
+
+// start update every 24 hours
+setInterval(() => {
+  LanaGrossaIntegrationInstance.RunFetcher();
+}, 1000 * 60 * 60 * 1);
+LanaGrossaIntegrationInstance.RunFetcher();
+
+server.get("/stats", async (request, reply) => {
+  const CurrentStats = LanaGrossaIntegrationInstance.CurrentStats;
+  const StatList = LanaGrossaIntegrationInstance.StatList;
+
+  const RemainingLinks = LanaGrossaIntegrationInstance.LinkQueue.length;
+
+  return {
+    RemainingLinks,
+    CurrentStats,
+    StatList,
+  };
+});
+
+server.listen({ port: 3010, host: "0.0.0.0" }, (err, address) => {
+  if (err) {
+    console.error(err);
+    process.exit(1);
+  }
+  console.log(`Server listening at ${address}`);
+});

src/types/supabase.ts

@@ -0,0 +1,191 @@
+export type Json =
+  | string
+  | number
+  | boolean
+  | null
+  | { [key: string]: Json | undefined }
+  | Json[]
+
+export type Database = {
+  public: {
+    Tables: {
+      Wool: {
+        Row: {
+          composition: string[] | null
+          created_at: string
+          id: string
+          key: string
+          maker: Database["public"]["Enums"]["WoolMaker"] | null
+          name: string
+          needle_size_max: number | null
+          needle_size_min: number | null
+          run_length: number | null
+          stitch_test_m: number | null
+          stitch_test_r: number | null
+          updated_at: string | null
+          weight: number | null
+        }
+        Insert: {
+          composition?: string[] | null
+          created_at?: string
+          id?: string
+          key: string
+          maker?: Database["public"]["Enums"]["WoolMaker"] | null
+          name?: string
+          needle_size_max?: number | null
+          needle_size_min?: number | null
+          run_length?: number | null
+          stitch_test_m?: number | null
+          stitch_test_r?: number | null
+          updated_at?: string | null
+          weight?: number | null
+        }
+        Update: {
+          composition?: string[] | null
+          created_at?: string
+          id?: string
+          key?: string
+          maker?: Database["public"]["Enums"]["WoolMaker"] | null
+          name?: string
+          needle_size_max?: number | null
+          needle_size_min?: number | null
+          run_length?: number | null
+          stitch_test_m?: number | null
+          stitch_test_r?: number | null
+          updated_at?: string | null
+          weight?: number | null
+        }
+        Relationships: []
+      }
+      WoolVariants: {
+        Row: {
+          color: string | null
+          created_at: string
+          ean: string | null
+          id: string
+          key: string | null
+          wool_id: string | null
+        }
+        Insert: {
+          color?: string | null
+          created_at?: string
+          ean?: string | null
+          id?: string
+          key?: string | null
+          wool_id?: string | null
+        }
+        Update: {
+          color?: string | null
+          created_at?: string
+          ean?: string | null
+          id?: string
+          key?: string | null
+          wool_id?: string | null
+        }
+        Relationships: [
+          {
+            foreignKeyName: "WoolVariants_wool_id_fkey"
+            columns: ["wool_id"]
+            isOneToOne: false
+            referencedRelation: "Wool"
+            referencedColumns: ["id"]
+          },
+        ]
+      }
+    }
+    Views: {
+      [_ in never]: never
+    }
+    Functions: {
+      [_ in never]: never
+    }
+    Enums: {
+      WoolMaker: "LanaGrossa"
+    }
+    CompositeTypes: {
+      [_ in never]: never
+    }
+  }
+}
+
+type PublicSchema = Database[Extract<keyof Database, "public">]
+
+export type Tables<
+  PublicTableNameOrOptions extends
+    | keyof (PublicSchema["Tables"] & PublicSchema["Views"])
+    | { schema: keyof Database },
+  TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
+    ? keyof (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
+        Database[PublicTableNameOrOptions["schema"]]["Views"])
+    : never = never,
+> = PublicTableNameOrOptions extends { schema: keyof Database }
+  ? (Database[PublicTableNameOrOptions["schema"]]["Tables"] &
+      Database[PublicTableNameOrOptions["schema"]]["Views"])[TableName] extends {
+      Row: infer R
+    }
+    ? R
+    : never
+  : PublicTableNameOrOptions extends keyof (PublicSchema["Tables"] &
+        PublicSchema["Views"])
+    ? (PublicSchema["Tables"] &
+        PublicSchema["Views"])[PublicTableNameOrOptions] extends {
+        Row: infer R
+      }
+      ? R
+      : never
+    : never
+
+export type TablesInsert<
+  PublicTableNameOrOptions extends
+    | keyof PublicSchema["Tables"]
+    | { schema: keyof Database },
+  TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
+    ? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
+    : never = never,
+> = PublicTableNameOrOptions extends { schema: keyof Database }
+  ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
+      Insert: infer I
+    }
+    ? I
+    : never
+  : PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
+    ? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
+        Insert: infer I
+      }
+      ? I
+      : never
+    : never
+
+export type TablesUpdate<
+  PublicTableNameOrOptions extends
+    | keyof PublicSchema["Tables"]
+    | { schema: keyof Database },
+  TableName extends PublicTableNameOrOptions extends { schema: keyof Database }
+    ? keyof Database[PublicTableNameOrOptions["schema"]]["Tables"]
+    : never = never,
+> = PublicTableNameOrOptions extends { schema: keyof Database }
+  ? Database[PublicTableNameOrOptions["schema"]]["Tables"][TableName] extends {
+      Update: infer U
+    }
+    ? U
+    : never
+  : PublicTableNameOrOptions extends keyof PublicSchema["Tables"]
+    ? PublicSchema["Tables"][PublicTableNameOrOptions] extends {
+        Update: infer U
+      }
+      ? U
+      : never
+    : never
+
+export type Enums<
+  PublicEnumNameOrOptions extends
+    | keyof PublicSchema["Enums"]
+    | { schema: keyof Database },
+  EnumName extends PublicEnumNameOrOptions extends { schema: keyof Database }
+    ? keyof Database[PublicEnumNameOrOptions["schema"]]["Enums"]
+    : never = never,
+> = PublicEnumNameOrOptions extends { schema: keyof Database }
+  ? Database[PublicEnumNameOrOptions["schema"]]["Enums"][EnumName]
+  : PublicEnumNameOrOptions extends keyof PublicSchema["Enums"]
+    ? PublicSchema["Enums"][PublicEnumNameOrOptions]
+    : never