Moved demo backend

Author: fgatti675

.yarn/install-state.gz

@@ -7,6 +7,4 @@ typings/
 
 node_modules/
 
-scripts
-
 .runtimeconfig.json

examples/example_pro/example_backend/.gitignore renamed to demo_backend/.gitignore

@@ -0,0 +1,10 @@
+{
+  "algolia": {
+    "search_key": "f084e6dcc154c04295c8124dbb797ff1",
+    "api_key": "6a014a04544a84a51e2f78d8246b6244",
+    "app_id": "Y6FR1MDSVW"
+  },
+  "mailchimp": {
+    "api_url": "https://us5.api.mailchimp.com/3.0/lists/d356ca5a84"
+  }
+}

examples/example_pro/example_backend/README.md renamed to demo_backend/README.md

@@ -14,23 +14,23 @@
     "main": "lib/index.js",
     "dependencies": {
         "@alpacahq/alpaca-trade-api": "^2.16.1",
-        "@elastic/elasticsearch": "^8.7.0",
-        "@google-cloud/firestore": "^6.4.1",
-        "algoliasearch": "^4.14.2",
-        "axios": "1.6.0",
+        "@elastic/elasticsearch": "^8.13.0",
+        "@google-cloud/firestore": "^6.8.0",
+        "algoliasearch": "^4.23.2",
+        "axios": "1.6.8",
         "cors": "^2.8.5",
-        "dotenv": "^16.0.3",
-        "express": "^4.18.2",
+        "dotenv": "^16.4.5",
+        "express": "^4.19.2",
         "fast-csv": "^4.3.6",
-        "firebase-admin": "^11.5.0",
-        "firebase-functions": "^4.2.1",
-        "firebase-tools": "^11.23.1",
+        "firebase-admin": "^11.11.1",
+        "firebase-functions": "^4.8.1",
+        "firebase-tools": "^11.30.0",
         "md5": "^2.3.0",
-        "meilisearch": "^0.32.3",
-        "typesense": "^1.5.3"
+        "meilisearch": "^0.32.5",
+        "typesense": "^1.7.2"
     },
     "devDependencies": {
-        "typescript": "^4.9.4"
+        "typescript": "^4.9.5"
     },
     "private": true
 }

examples/example_pro/example_backend/firebase.json renamed to demo_backend/firebase.json

@@ -0,0 +1,32 @@
+import * as admin from "firebase-admin";
+import { initServiceAccountFirestore } from "./util";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+
+function getRandomInt(max: number) {
+    return Math.floor(Math.random() * max);
+}
+
+const addLikes = async () => {
+    const productsIds = await firestore
+        .collection("products")
+        .get().then(snapshot => snapshot.docs.map(d => d.ref));
+    firestore
+        .collection("users")
+        .get()
+        .then((snapshot) => {
+            snapshot.docs.forEach((doc) => {
+                const count = getRandomInt(4);
+                const products = Array.from({ length: count })
+                    .fill(0)
+                    .map((_) => productsIds[getRandomInt(productsIds.length)]);
+                // console.log(products);
+                doc.ref.update({ liked_products: products });
+            });
+        });
+};
+addLikes();
+
+

examples/example_pro/example_backend/functions/.env.template renamed to demo_backend/functions/.env.template

@@ -0,0 +1,44 @@
+import * as admin from "firebase-admin";
+import * as dotenv from "dotenv";
+
+dotenv.config();
+
+import { initServiceAccountFirestore } from "./util";
+import { addUserToMailchimp } from "../src/mailchimp";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+
+const domains = new Set()
+// const startDate = new Date(2022, 1, 1);
+// list all users with the firebase admin sdk
+admin.auth().listUsers().then(async (listUsersResult) => {
+
+    // add user email domain to set
+    listUsersResult.users.forEach((userRecord) => {
+        if(userRecord?.email)
+        domains.add(userRecord?.email.split("@")[1]);
+    });
+    console.log(domains);
+    console.log(domains.size)
+
+    // const result = listUsersResult.users.filter((userRecord) => {
+    //     const date = new Date(Date.parse(userRecord.metadata.creationTime));
+    //     return date < startDate;
+    // }).sort((a, b) => {
+    //     return Date.parse(b.metadata.creationTime) - Date.parse(a.metadata.creationTime);
+    // });
+    // for (const userRecord of result) {
+    //     const date = new Date(Date.parse(userRecord.metadata.creationTime));
+    //     console.log("user", userRecord.email, date);
+    //     try {
+    //         if (userRecord.email)
+    //             await addUserToMailchimp(userRecord.email, "import");
+    //     } catch (e) {
+    //         console.error(userRecord.email);
+    //     }
+    // }
+});
+
+//

examples/example_pro/example_backend/functions/.gitignore renamed to demo_backend/functions/.gitignore

@@ -0,0 +1,6 @@
+import * as dotenv from 'dotenv';
+import { fetchDemoData } from "../src/alpaca";
+
+dotenv.config();
+
+fetchDemoData().then(console.log);

demo_backend/functions/.runtimeconfig.json

@@ -0,0 +1,14 @@
+import * as admin from "firebase-admin";
+
+import { initServiceAccountFirestore } from "./util";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+firestore.collection("/blog")
+    .where("status", "==", "draft")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            d.ref.delete();
+        }));

demo_backend/functions/.yarn/install-state.gz

@@ -0,0 +1,107 @@
+import * as admin from "firebase-admin";
+import { initServiceAccountFirestore } from "./util";
+const crypto = require('crypto');
+
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+
+// const categories = new Set<string>();
+// const catMap: Record<string, any> = {};
+
+
+const categories = {
+    fiction: "Fiction",
+    drama: "Drama",
+    "fantasy-fiction": "Fantasy fiction",
+    history: "History",
+    religion: "Religion",
+    "self-help": "Self-Help",
+    "comics-graphic-novels": "Comics & Graphic Novels",
+    "juvenile-fiction": "Juvenile Fiction",
+    philosophy: "Philosophy",
+    fantasy: "Fantasy",
+    education: "Education",
+    science: "Science",
+    medical: "Medical",
+    cooking: "Cooking",
+    travel: "Travel"
+};
+
+const generateBooks = async () => {
+    const fs = require("fs")
+    const csv = require("fast-csv");
+    const data: any[] = []
+
+    fs.createReadStream("books.csv")
+        .pipe(csv.parse({ headers: true }))
+        .on("error", (error: any) => console.error(error))
+        .on("data", (row: any) => data.push(row))
+        .on("end", () => {
+                data.sort((a: any, b: any) => {
+                    return parseFloat(b.average_rating) - parseFloat(a.average_rating)
+                });
+                const books = data
+                    .filter(b => parseFloat(b.ratings_count) > 1000)
+                    .slice(0, 200)
+                    .map((b) => {
+                        const catSlugs = b.categories.split(",").map((c: string) => slugify(c.trim()));
+                        const cat = catSlugs.find((c: string) => c in categories);
+                        let newVar = {
+                            isbn13: parseInt(b.isbn13),
+                            isbn10: parseInt(b.isbn10),
+                            title: b.title,
+                            subtitle: b.subtitle,
+                            authors: b.authors,
+                            thumbnail: b.thumbnail,
+                            description: b.description,
+                            published_year: parseInt(b.published_year),
+                            average_rating: parseFloat(b.average_rating),
+                            num_pages: parseInt(b.num_pages),
+                            ratings_count: parseInt(b.ratings_count),
+                        };
+                        if (cat) {
+                            // @ts-ignore
+                            newVar.category = cat;
+                        }
+                        return newVar;
+                    });
+
+                books.forEach((b: any) => {
+                    const hash = crypto.createHash('sha256', "sdfcerfewrf")
+                        .update(b.title)
+                        .digest('hex');
+                    firestore
+                        .collection("books")
+                        .doc(hash)
+                        // .doc(b.isbn13.toString())
+                        .set(b);
+                });
+                console.log(books);
+            }
+        );
+
+
+};
+generateBooks();
+
+const slugify = function (text?: string) {
+    if (!text) return "";
+    const from = "ãàáäâẽèéëêìíïîõòóöôùúüûñç·/_,:;"
+    const to = "aaaaaeeeeeiiiiooooouuuunc------"
+
+    for (var i = 0, l = from.length; i < l; i++) {
+        text = text.replace(new RegExp(from.charAt(i), "g"), to.charAt(i));
+    }
+
+    return text
+        .toString()                     // Cast to string
+        .toLowerCase()                  // Convert the string to lowercase letters
+        .trim()                         // Remove whitespace from both sides of a string
+        .replace(/\s+/g, "-")           // Replace spaces with -
+        .replace(/&/g, "-")             // Replace & with -
+        .replace(/[^\w\-]+/g, "")       // Remove all non-word chars
+        .replace(/\-\-+/g, "-")         // Replace multiple - with single -
+        .replace(/^\s+|\s+$/g, "");
+}

examples/example_pro/example_backend/functions/package.json renamed to demo_backend/functions/package.json

@@ -0,0 +1,29 @@
+import * as admin from "firebase-admin";
+import axios from "axios";
+import { initServiceAccountFirestore } from "./util";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+
+const generateUsers = async () => {
+    const url = `https://randomuser.me/api/?results=500`;
+    const response = await axios.get(url);
+    console.log(response);
+    response.data.results.forEach((data: any) => firestore
+        .collection("users")
+        .doc()
+        .set({
+            email: data.email,
+            gender: data.gender,
+            first_name: data.name.first,
+            last_name: data.name.last,
+            picture: data.picture,
+            location: data.location,
+            phone: data.phone
+        }));
+
+};
+generateUsers();
+
+

demo_backend/functions/scripts/addLikedProducts.ts

@@ -0,0 +1,30 @@
+import * as admin from "firebase-admin";
+import * as dotenv from "dotenv";
+
+dotenv.config();
+import {initServiceAccountFirestore} from "./util";
+import {indexInAlgolia} from "../src/indexing/algolia";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+firestore.collection("/blog")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInAlgolia("blog", d.data(), d.id);
+        }));
+
+firestore.collection("/users")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInAlgolia("users", d.data(), d.id);
+        }));
+
+firestore.collection("products")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInAlgolia("products", d.data(), d.id);
+        }));

demo_backend/functions/scripts/addMailchimpUsers.ts

@@ -0,0 +1,33 @@
+import * as admin from "firebase-admin";
+import * as dotenv from "dotenv";
+
+dotenv.config();
+import { initServiceAccountFirestore } from "./util";
+import { indexInElasticSearch } from "../src/indexing/elasticsearch";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+// firestore.collection("/blog")
+//     .get()
+//     .then(async (snapshot) => {
+//         for (const d of snapshot.docs) {
+//             await indexInElasticSearch("blog", d.data(), d.id);
+//         }
+//     });
+
+firestore.collection("/users")
+    .get()
+    .then(async (snapshot) => {
+        for (const d of snapshot.docs) {
+            await indexInElasticSearch("users", d.data(), d.id);
+        }
+    });
+//
+// firestore.collection("products")
+//     .get()
+//     .then(async (snapshot) => {
+//         for (const d of snapshot.docs) {
+//             await indexInElasticSearch("products", d.data(), d.id);
+//         }
+//     });

demo_backend/functions/scripts/alpaca.ts

@@ -0,0 +1,31 @@
+import * as admin from "firebase-admin";
+import * as dotenv from "dotenv";
+
+dotenv.config();
+
+import { initServiceAccountFirestore } from "./util";
+import { indexInMeiliSearch } from "../src/meilisearch";
+
+initServiceAccountFirestore(true);
+
+const firestore = admin.firestore();
+firestore.collection("/blog")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInMeiliSearch("blog", d.data(), d.id);
+        }));
+
+firestore.collection("/users")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInMeiliSearch("users", d.data(), d.id);
+        }));
+
+firestore.collection("products")
+    .get()
+    .then((snapshot) =>
+        snapshot.docs.forEach(d => {
+            indexInMeiliSearch("products", d.data(), d.id);
+        }));