initial commit

.env

@@ -0,0 +1,4 @@
+REDIS_HOST='127.0.0.1'
+REDIS_PORT=6379
+REDIS_PASSWORD=pinguin
+APPLICATION_ROLE='queue'

.gitignore

@@ -0,0 +1,3 @@
+.idea
+node_modules
+lib

Dockerfile

@@ -0,0 +1,10 @@
+FROM node:20
+
+COPY . /usr/src/app/
+
+WORKDIR /usr/src/app
+RUN corepack enable pnpm
+RUN pnpm install
+RUN pnpm run build
+
+CMD ["npm", "start"]

README.md

@@ -0,0 +1,48 @@
+# Pinguin
+
+Pinguin is a simple service for monitoring websites. It is written in Node.js and uses Redis as message broker.
+Its composed of two parts: ingress and worker. Ingress is responsible for receiving HTTP requests and forwards them to
+workers through BullMQ.
+
+## Build instructions
+
+```bash
+# build an image
+docker build -t haxagon/pinguin .
+
+# push an image to private registry
+docker push haxagon/pinguin
+```
+
+## Development
+
+```bash
+# install dependencies
+npm install
+
+# build
+npm run build
+
+# start local redis server
+docker run -d --name redis -p 127.0.0.1:6379:6379 --rm redis:latest
+```
+
+### Start Ingress
+
+```bash
+APPLICATION_ROLE='ingress' ./node_modules/nodemon/bin/nodemon.js lib/bradlo.js
+```
+
+### Start Worker
+
+```bash
+APPLICATION_ROLE='worker' ./node_modules/nodemon/bin/nodemon.js lib/bradlo.js
+```
+
+## API
+
+### POST /ping
+
+```bash
+curl --json '{"domain": "seznam.cz"}' -X POST localhost:3000/ping 
+```

package.json

@@ -0,0 +1,37 @@
+{
+  "name": "pinguin",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "./node_modules/typescript/bin/tsc",
+    "start": "node ./lib/app.js",
+    "lint:fix": "eslint . --fix"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/express": "^4.17.18",
+    "@types/node": "^20.7.1",
+    "@typescript-eslint/eslint-plugin": "^6.7.3",
+    "@typescript-eslint/parser": "^6.7.3",
+    "eslint": "^8.50.0",
+    "eslint-config-prettier": "^9.0.0",
+    "ioredis": "^5.3.2",
+    "nodemon": "^3.0.1",
+    "prettier": "^3.0.3",
+    "typescript": "^5.2.2"
+  },
+  "dependencies": {
+    "bullmq": "^4.12.0",
+    "chalk": "^5.3.0",
+    "dotenv": "^16.3.1",
+    "express": "^4.18.2",
+    "express-validator": "^7.0.1",
+    "node-fetch": "^3.3.2",
+    "pino": "^6.14.0",
+    "pino-pretty": "^10.2.0"
+  },
+  "type": "module"
+}

src/api/index.ts

@@ -0,0 +1,55 @@
+import express from 'express';
+import {body, validationResult} from 'express-validator';
+import {logger} from '../services/logger.js';
+import {sendPingTask} from "../services/queue.js";
+
+const api = express();
+api.use(express.json());
+const pingValidation = [
+    body('domain')
+        .exists()
+        .withMessage('domain is required')
+        .isString()
+];
+api.post('/ping', pingValidation, async (req, res) => {
+    const validationErrors = validationResult(req);
+    if (!validationErrors.isEmpty()) {
+        logger.warn('invalid domain ping request received', {
+            Data: req.body,
+            Errors: validationErrors.array()
+        });
+        return res.json(validationErrors.array());
+    }
+    try {
+        const pingResult = await sendPingTask(req.body.domain);
+        return res.json({"result": pingResult});
+    } catch (e) {
+        logger.error({
+            Data: req.body,
+            err: e
+        }, 'error ping domain');
+        return res.status(500).json({error: e.message});
+    }
+});
+
+// in case I want to remotely poweroff the server
+api.post('/poweroff', async (req, res) => {
+    logger.info(`poweroff request received`)
+    try {
+        process.exit(0)
+        return res.json({"result": "OK"});
+    } catch (e) {
+        logger.error({
+            Data: req.body,
+            err: e
+        }, 'error poweroff');
+        return res.status(500).json({error: e.message});
+    }
+})
+
+export function startApiServer() {
+    logger.info('starting API server');
+    api.listen(3000, () => {
+        logger.info('API server listening on port 3000');
+    });
+}

src/app.ts

@@ -0,0 +1,20 @@
+import dotenv from 'dotenv';
+import {startApiServer} from './api/index.js';
+import {config} from "./config.js";
+import {createPingQueue} from "./services/queue.js";
+import {startPingWorker} from "./services/worker.js";
+import {logger} from "./services/logger.js";
+
+(async () => {
+    dotenv.config();
+    logger.info(`starting pinguin with application role ${config.applicationRole}`)
+    switch (config.applicationRole) {
+        case "queue":
+            await createPingQueue();
+            startApiServer()
+            break;
+        case "worker":
+            await startPingWorker();
+            break;
+    }
+})();

src/config.ts

@@ -0,0 +1,27 @@
+import dotenv from "dotenv";
+import {ConnectionOptions} from "bullmq";
+
+export interface PinguinConfig {
+    redis: {
+        host: string
+        port: number
+    }
+    applicationRole: "worker" | "queue"
+}
+
+// use dotenv to load .env variables to config (mainly for development)
+dotenv.config()
+
+export const config: PinguinConfig = {
+    redis: {
+        host: process.env.REDIS_HOST || 'localhost',
+        port: parseInt(process.env.REDIS_PORT) || 6379
+    },
+    applicationRole: process.env.APPLICATION_ROLE as "worker" | "queue"
+}
+
+export const redisConnectionConfig: ConnectionOptions = {
+    host: config.redis.host,
+    port: config.redis.port,
+    password: process.env.REDIS_PASSWORD
+}

src/services/executor.ts

@@ -0,0 +1,8 @@
+import {exec as childProcessExec} from "child_process";
+import * as util from "util";
+const exec = util.promisify(childProcessExec)
+
+export const execCommand = async (command: string) => {
+    const commandResult = await exec(command)
+    return commandResult.stdout
+}

src/services/logger.ts

@@ -0,0 +1,9 @@
+import pino from 'pino';
+
+export const logger = pino({
+    name: 'bradlo',
+    level: 'debug',
+    prettyPrint: {
+        colorize: true
+    }
+});

src/services/queue.ts

@@ -0,0 +1,28 @@
+import {redisConnectionConfig} from "../config.js";
+import {Job, Queue, QueueEvents} from "bullmq";
+import {logger} from "./logger.js";
+
+let queue: Queue
+let queueEvents: QueueEvents
+export const createPingQueue = async () => {
+    logger.info(`starting ping queue`)
+    queue = new Queue('ping', {
+        connection: redisConnectionConfig
+    })
+    logger.info(redisConnectionConfig)
+    queueEvents = new QueueEvents('ping', {
+        connection: redisConnectionConfig
+    })
+
+    await queue.waitUntilReady()
+    await queueEvents.waitUntilReady()
+    logger.info(`ping queue started`)
+}
+export const sendPingTask = async (domain: string) => {
+    logger.info(`adding ping task to queue to ping domain ${domain}`)
+    const job = await queue.add('ping', {domain})
+    logger.info(`ping task added to queue to ping domain ${domain}`)
+    await job.waitUntilFinished(queueEvents)
+    const returnedJob = await Job.fromId(queue, job.id)
+    return returnedJob.returnvalue
+}

src/services/worker.ts

@@ -0,0 +1,31 @@
+import {Worker} from 'bullmq';
+import {logger} from "./logger.js";
+import {execCommand} from "./executor.js";
+import {redisConnectionConfig} from "../config.js";
+
+interface PingTaskData {
+    domain: string
+}
+
+// starts the worker which picks tasks from the bullmq queue and executes them
+export const startPingWorker = async () => {
+    logger.info(`starting ping worker`)
+    const worker = new Worker<PingTaskData>('ping', async job => {
+        if (!job.data.domain) {
+            const error = new Error(`Invalid job ${job.id} received from queue ${worker.name}. Domain is missing in job data.`)
+            logger.error(error);
+            throw error;
+        }
+        logger.info(`worker retrieved job ${job.id} from queue ${worker.name} with id ${job.id} and domain to ping ${job.data.domain}`);
+        const result = await execCommand(`ping -c 1 ${job.data.domain}`)
+        logger.info(`worker finished job ${job.id} from queue ${worker.name} with id ${job.id} and domain to ping ${job.data.domain} with result ${result}`);
+        return result
+    }, {
+        connection: redisConnectionConfig
+    });
+    worker.on('failed', (job, err) => {
+        logger.error(`worker failed job ${job.id} from queue ${worker.name} with id ${job.id} and domain to ping ${job.data.domain} with error ${err}`);
+    })
+    await worker.waitUntilReady();
+    logger.info(`ping worker started and waiting for jobs`)
+}

tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "node16",
+    "lib": [
+      "ES2022"
+    ],
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "rootDir": "./src",
+    "outDir": "lib",
+    "allowSyntheticDefaultImports": true,
+    "importHelpers": true,
+    "alwaysStrict": true,
+    "sourceMap": true,
+    "forceConsistentCasingInFileNames": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitReturns": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitAny": false,
+    "noImplicitThis": false,
+    "strictNullChecks": false,
+    "skipLibCheck": true
+  },
+  "include": [
+    "src/**/*"
+  ]
+}