improved docker deployment

Author: NoxelS

.env_template renamed to .env.template

@@ -1,9 +1,9 @@
 # Settings for database
-HOST=<host of dev database>
-USER=<user for dev database>
-PASSWORD=<password for dev database>
-DATABASE=<database of dev database>
-PORT=<port of dev database>
+DB_HOST=<host of dev database>
+DB_USER=<user for dev database>
+DB_PASSWORD=<password for dev database>
+DB_DATABASE=<database of dev database>
+DB_PORT=<port of dev database>
 
 # Settings for email notification
 MAIL_HOST=<host>
@@ -17,6 +17,6 @@ MAIL_SUBJECT=<subject>
 # Openai api key (https://openai.com)
 OPENAI_API_KEY=<openai api key>
 
-
 # 2captcha api key (https://2captcha.com)
-TWO_CAPTCHA_API_KEY=<2captcha api key>
+TWO_CAPTCHA_API_KEY=<2captcha api key>
+

.gitignore

@@ -73,6 +73,8 @@ typings/
 
 # dotenv environment variables file
 .env
+database.env
+phpmyadmin.env
 .env.test
 
 # parcel-bundler cache (https://parceljs.org/)

Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update \
     && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
     && apt-get update \
     && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
-    --no-install-recommends \
+      --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 # If running Docker >= 1.13.0 use docker run's --init arg to reap zombie processes, otherwise
@@ -21,25 +21,31 @@ RUN apt-get update \
 # Uncomment to skip the chromium download when installing puppeteer. If you do,
 # you'll need to launch puppeteer with:
 #     browser.launch({executablePath: 'google-chrome-stable'})
-# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-WORKDIR /usr/src/app
-COPY package*.json ./
-RUN npm install
-RUN npm ci --only=production
-COPY . .
+# ENV PUPPETEER_SKIP_DOWNLOAD true
 
 # Install puppeteer so it's available in the container.
-    # Add user so we don't need --no-sandbox.
-    # same layer as npm install to keep re-chowned files from using up several hundred MBs more space
-# RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
+# RUN npm init -y &&  \
+#     npm i puppeteer \
+#     # Add user so we don't need --no-sandbox.
+#     # same layer as npm install to keep re-chowned files from using up several hundred MBs more space
+#     && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
 #     && mkdir -p /home/pptruser/Downloads \
 #     && chown -R pptruser:pptruser /home/pptruser \
 #     && chown -R pptruser:pptruser /node_modules \
 #     && chown -R pptruser:pptruser /package.json \
 #     && chown -R pptruser:pptruser /package-lock.json
 
+WORKDIR /usr/src/app
+COPY package*.json ./
+RUN npm install
+RUN npm ci --only=production
+COPY . .
+
 # Run everything after as non-privileged user.
 # USER pptruser
 
-CMD [ "npm", "start" ]
+ENV NODE_ENV=production
+
+RUN npm run compile
+
+CMD [ "npm", "run", "start:prod" ]

README.md

@@ -4,11 +4,12 @@ This advanced TypeScript Puppeteer web scraper template offers a comprehensive s
 
 ## Features
 
-- **Environment-Specific Configuration**: Leverages `.env` files for differentiating between development and production environments.
 - **Puppeteer Plugins Integration**: Includes plugins like `puppeteer-extra-plugin-anonymize-ua`, `puppeteer-extra-plugin-adblocker`, `puppeteer-extra-plugin-recaptcha`, and `puppeteer-extra-plugin-stealth` for enhanced scraping capabilities.
 - **Automated Scheduling**: Utilizes `node-cron` for scheduling scraping tasks, customizable for different intervals.
+- **Environment-Specific Configuration**: Leverages `.env` files for differentiating between development and production environments.
 - **MySQL Database Integration**: Features integration with MySQL using a connection pool for efficient data handling.
 - **Error Handling and Debugging**: Advanced error handling with screenshot capabilities for debugging, along with options to open devtools and slow down Puppeteer operations for detailed inspection.
+- **Automated Deployment**: Includes a docker-compose file for automated deployment of the scraper. This will automatically build the scraper, a MySQL database, and a phpMyAdmin instance for database management.
 
 ## Getting Started
 
@@ -31,10 +32,10 @@ This advanced TypeScript Puppeteer web scraper template offers a comprehensive s
 
 ### Configuration
 
-1. Create a `.env` file in the root directory.
-2. Add the necessary environment variables (as declared in the `.env.example` file) to the `.env` file or environment variables (recommended).
+1. Create thre three env files `.env`, `database.env` and `phpmyadmin.env` in the root directory.
+2. Add the necessary environment variables (as declared in the `*.env.example` files) to the `.env` files or environment variables.
 
-### Usage
+### Local Usage
 
 - Compile the scraper:
 
@@ -52,6 +53,15 @@ This advanced TypeScript Puppeteer web scraper template offers a comprehensive s
     npm start
     ```
 
+### Docker Usage
+
+- Build the scraper, MySQL database, and phpMyAdmin instance:
+
+    ```sh
+    docker-compose up
+    ```
+    Make sure to add the necessary environment variables to the `database.env` and `phpmyadmin.env` files.
+
 ## TypeScript and Puppeteer Integration
 
 - **TypeScript Support**: Fully supported with TypeScript for type safety and easier code management.

data/readme.md

@@ -0,0 +1,2 @@
+This folder is mounted inside a volume, so it will be persisted even if the container is removed. This is useful for storing data
+that you want to keep between container restarts but don't want to store in the database.

database.env.template

@@ -0,0 +1,5 @@
+MYSQL_ROOT_PASSWORD=
+MYSQL_USER=
+MYSQL_PASSWORD=
+MYSQL_DATABASE=
+DB_PORT=

docker-compose.yml

@@ -0,0 +1,44 @@
+version: '2'
+
+volumes:
+  scraper-db-data:
+  scraper-data:
+
+services:
+  scraper:
+    container_name: scraper
+    build: ./
+    depends_on:
+      - db
+    env_file:
+      - .env
+    volumes:
+      - scraper-data:/usr/src/app/data
+
+  phpmyadmin:
+    container_name: phpmyadmin
+    image: phpmyadmin/phpmyadmin
+    depends_on:
+      - db
+    ports:
+      - 9999:80
+    expose:
+      - 9999
+    restart: always
+    env_file:
+      - database.env
+      - phpmyadmin.env
+
+  db:
+    container_name: scraper-db
+    image: mysql:latest
+    command:
+      --default-authentication-plugin=mysql_native_password
+    env_file:
+      - database.env
+    volumes:
+      - scraper-db-data:/var/lib/mysql
+    ports:
+      - 7706:3306
+    expose:
+      - 7706

phpmyadmin.env.template

@@ -0,0 +1,3 @@
+PMA_HOST=
+PMA_PORT=
+PMA_ARBITRARY=

src/scraper.ts

@@ -1,9 +1,11 @@
 import { config } from 'dotenv';
 import { createPool, Pool, PoolConfig } from 'mysql';
 import { schedule } from 'node-cron';
-import { LaunchOptions } from 'puppeteer';
+import { PuppeteerLaunchOptions } from 'puppeteer';
 import Puppeteer from 'puppeteer-extra';
 
+import { query } from './storage';
+
 
 /** Only use .env files when running in dev mode */
 const isProduction = process.env.production?.toString() === 'true' || process.env.NODE_ENV === 'production';
@@ -21,13 +23,13 @@ Puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
 Puppeteer.use(StealthPlugin());
 
 /** Launch options */
-const launchOptions: LaunchOptions = {
+const launchOptions: PuppeteerLaunchOptions  = {
     headless: isProduction, // Run headless in production mode
     args: [
         '--disable-gpu', '--disable-dev-shm-usage', '--disable-setuid-sandbox', '--no-sandbox',
         '--window-size=1920,1080', /* '--window-position=1920,0' */ // Activate this if you want to have the browser window on a second screen
     ],
-    ignoreHTTPSErrors: true, 
+    ignoreHTTPSErrors: true, // Ignore HTTPS errors
     devtools: !isProduction, // Open devtools in development mode
     slowMo: 0, // Slow down puppeteer operations by X milliseconds (useful for debugging)
     timeout: 0 // Disable timeouts
@@ -67,7 +69,7 @@ async function scrape(pool: Pool) {
     } catch (error) {
         if (error instanceof Error) {
             /** Save a screenshot if possible */
-            try { await page.screenshot({ path: `log/err-${new Date().getTime()}.png` }) } catch (error) { }
+            try { await page.screenshot({ path: `data/err-${new Date().getTime()}.png` }) } catch (error) { }
             console.error(error.message);
         }
     }
@@ -77,20 +79,23 @@ async function scrape(pool: Pool) {
 
 /** Create MySQL connection pool so we can reuse connections */
 const pool: Pool = createPool(<PoolConfig>{
-    host: process.env.HOST,
-    user: process.env.USER,
-    password: process.env.PASSWORD,
-    database: process.env.DATABASE,
-    port: process.env.PORT
+    host: process.env.DB_HOST,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASSWORD,
+    database: process.env.DB_DATABASE,
+    port: process.env.DB_PORT
 });
 
+/* Test connection */
+query('SHOW TABLES FROM data;', [], (e, r) => {console.log(e ? e : `You have the following tables: ${r[0]}`);}, pool);
+
 /*
  * Scrape every 15 minutes if production mode is enabled or once
  * if not.
  * (https://crontab.guru is your best friend)
  */
-const interval = process.env.production ? '*/30 * * * *' : '* * * * *';
-console.log(`Scraping ${process.env.production ? 'every 30 minutes' : 'once.'}.`);
+const interval = isProduction ? '*/30 * * * *' : '* * * * *';
+console.log(`Scraping ${isProduction ? 'every 30 minutes' : 'once'} in ${isProduction ? 'production' : 'dev'} mode.`);
 
-if (!process.env.production) scrape(pool)
-else schedule(interval, () => scrape(pool));
+if (isProduction) schedule(interval, () => scrape(pool));
+else scrape(pool);