Merge pull request #65 from telefonicaid/feature/cluster-support-prelanding

Update IoTA-Sigfox with the changes of the IoTA Node Lib related to multi-threading (final landing)

Author: AlvaroVega

CHANGES_NEXT_RELEASE

@@ -1,2 +1,9 @@
-- ADD PM2_ENABLED flag to Docker
-- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues
+- ADD PM2_ENABLED flag to Docker.
+- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues.
+- Update sigfox-test.js to show the status of the response after send a message.
+- Improve MandatoryFieldsNotFound function (errors.js) in order to incorporate which mandatory fields is
+not found.
+- Improve in requiredFields function (sigfoxHandlers.js) in order to know the exactly field that it is not
+provided.
+- NOTE: in the "upgrade iotagent-node-lib from X to Y" include a mention to configure multithreading using cluster
+nodejs module (through multiCore config.js parameter or IOTA_MULTI_CORE environment variable).

bin/iotagent-sigfox

@@ -25,6 +25,8 @@
 'use strict';
 
 var iotAgent = require('../lib/iotagentCore'),
+    iotAgentLib = require('iotagent-node-lib'),
+    info = require('../package.json'),
     context = {
         op: 'IoTAgentSIGFOX.Executable'
     },
@@ -39,7 +41,9 @@ function start() {
         config = require('../config');
     }
 
-    iotAgent.start(config, function (error) {
+    config.iota.iotaVersion = info.version;
+
+    iotAgentLib.startServer(config, iotAgent, function (error) {
         if (error) {
             logger.error(context, 'Error starting Sigfox IoT Agent: [%s] Exiting process',
                          JSON.stringify(error));

bin/sigfox-test.js

@@ -57,6 +57,9 @@ function sendMeasure(commands) {
             console.log('\nError sending data to the Sigfox IoT Agent: ' + error);
         } else {
             console.log('\nData successfully sent');
+
+            // Now, we analyse the response obtained in the call
+            console.log('Response Code [%d]: [%s]', response.statusCode, JSON.parse(body)['message']);
         }
 
         clUtils.prompt();

config.js

@@ -135,7 +135,8 @@ config.iota = {
 
     /**
      * Configuration for the IoT Manager. If the IoT Agent is part of a configuration composed of multiple IoTAgents
-     * coordinated by an IoT Manager, this section defines the information that will be used to connect with that manager.
+     * coordinated by an IoT Manager, this section defines the information that will be used to connect with that
+     * manager.
      */
     iotManager: {
         /**
@@ -252,4 +253,10 @@ config.iota = {
  */
 config.defaultKey = 'TEF';
 
+/**
+ * flag indicating whether the node server will be executed in multi-core option (true) or it will be a
+ * single-thread one (false).
+ */
+//config.multiCore = false;
+
 module.exports = config;

docker/README.md

@@ -180,3 +180,42 @@ Currently, this `_FILE` suffix is supported for:
 -   `IOTA_AUTH_PASSWORD`
 -   `IOTA_AUTH_CLIENT_ID`
 -   `IOTA_AUTH_CLIENT_SECRET`
+
+## Best Practices
+
+### Increase ULIMIT in Docker Production Deployments
+
+Default settings for ulimit on a Linux system assume that several users would share the system. These settings limit the
+number of resources used by each user. The default settings are generally very low for high performance servers and
+should be increased. By default, we recommend, that the SigFox IoTAgent server in high performance scenarios, the
+following changes to ulimits:
+
+```console
+ulimit -n 65535        # nofile: The maximum number of open file descriptors (most systems do not allow this
+                                 value to be set)
+ulimit -c unlimited    # core: The maximum size of core files created
+ulimit -l unlimited    # memlock: The maximum size that may be locked into memory
+```
+
+If you are just doing light testing and development, you can omit these settings, and everything will still work.
+
+To set the ulimits in your container, you will need to run SigFox IoTAgent Docker containers with the following
+additional --ulimit flags:
+
+```console
+docker run --ulimit nofile=65535:65535 --ulimit core=100000000:100000000 --ulimit memlock=100000000:100000000 \
+--name iotagent -d fiware/sigfox-iotagent
+```
+
+Since “unlimited” is not supported as a value, it sets the core and memlock values to 100 GB. If your system has more
+than 100 GB RAM, you will want to increase this value to match the available RAM on the system.
+
+> Note: The --ulimit flags only work on Docker 1.6 or later.
+
+Nevertheless, you have to "request" more resources (i.e. multiple cores), which might be more difficult for orchestrates
+([Docker Engine](https://docs.docker.com/engine) or [Kubernetes](https://kubernetes.io)) to schedule than a few
+different containers requesting one core (or less...) each (which it can, in turn, schedule on multiple nodes, and not
+necessarily look for one node with enough available cores).
+
+If you want to get more details about the configuration of the system and node.js for high performance scenarios, please
+refer to the [Installation Guide](https://iotagent-sigfox.readthedocs.io/en/latest/installationguide/index.html).

docs/installationguide.md

@@ -1,3 +1,8 @@
+# Installation & Administration Guide
+
+-   [Installation](#installation)
+-   [High performance configuration usage](#high-performance-configuration-usage)
+
 ## Installation
 
 The Sigfox IoT Agent can be installed in two ways: cloning the GitHub repository, or using Docker.
@@ -26,3 +31,147 @@ Sigfox IoT Agent:
 ```bash
 docker run -t -i --link orion:orion -p 4041:4041 -p 17428:17428 fiware/sigfox-iotagent
 ```
+
+## High performance configuration usage
+
+Node.js is single‑threaded and uses non-blocking I/O, allowing it to scale up to tens of thousands of concurrent
+operations. Nevertheless, Node.js has a few weak points and vulnerabilities that can make Node.js‑based systems to offer
+under-performance behaviour, specially when a Node.js web application experiences rapid traffic growth.
+
+Additionally, It is important to know the place in which the node.js server is running, because it has limitations.
+There are two types of limits on the host: hardware and software. Hardware limits can be easy to spot. Your application
+might be consuming all of the memory and needing to consume disk to continue working. Adding more memory by upgrading
+your host, whether physical or virtual, seems to be the right choice.
+
+Moreover, Node.js applications have also a software memory limit (imposed by V8), therefore we cannot forget about these
+limitations when we execute a service. In this case of 64-bit environment, your application would be running by default
+at a 1 GB V8 limit. If your application is running in high traffic scenarios, you will need a higher limit. The same is
+applied to other parameters.
+
+It means that we need to make some changes in the execution of node.js and in the configuration of the system:
+
+-   **Node.js flags**
+
+    -   **--use-idle-notification**
+
+        Turns of the use idle notification to reduce memory footprint.
+
+    -   **--expose-gc**
+
+        Use the expose-gc command to enable manual control of the garbage collector from the own node.js server code. In
+        case of the IoTAgent, it is not implemented because it is needed to implement the calls to the garbage collector
+        inside the ser server, nevertheless the recommended value is every 30 seconds.
+
+    -   **--max-old-space-size=xxxx**
+
+        In that case, we want to increase the limit for heap memory of each V8 node process in order to use max capacity
+        that it is possible instead of the 1,4Gb default on 64-bit machines (512Mb on a 32-bit machine). The
+        recommendation is at least to use half of the total memory of the physical or virtual instance.
+
+-   **User software limits**
+
+    Linux kernel provides some configuration about system related limits and maximums. In a distributed environment with
+    multiple users, usually you need to take into control the resources that are available for each of the users.
+    Nevertheless, when the case is that you have only one available user but this one request a lot of resources due to
+    a high performance application the default limits are not proper configured and need to be changed to resolve the
+    high performance requirements. These are like maximum file handler count, maximum file locks, maximum process count
+    etc.
+
+    You can see the limits of your system executing the command:
+
+    ```bash
+    ulimit -a
+    ```
+
+    You can define the corresponding limits inside the file limits.conf. This description of the configuration file
+    syntax applies to the `/etc/security/limits.conf` file and \*.conf files in the `/etc/security/limits.d` directory.
+    You can get more information about the limits.conf in the
+    [limits.con - linux man pages](http://man7.org/linux/man-pages/man5/limits.conf.5.html). The recommended values to
+    be changes are the following:
+
+    -   **core**
+
+        Limits of the core file size in KB, we recommend to change to `unlimited` both hard and soft types.
+
+            * soft core unlimited
+            * hard core unlimited
+
+    -   **data**
+
+        Maximum data size in KB, we recommend to change to `unlimited` both hard and soft types.
+
+            * soft data unlimited
+            * hard data unlimited
+
+    -   **fsize**
+
+        Maximum filesize in KB, we recommend to change to `unlimited` both hard and soft types.
+
+            * soft fsize unlimited
+            * hard fsize unlimited
+
+    -   **memlock**
+
+        Maximum locked-in-memory address space in KB, we recommend to change to `unlimited` both hard and soft types.
+
+            * memlock unlimited
+            * memlock unlimited
+
+    -   **nofile**
+
+        Maximum number of open file descriptors, we recommend to change to `65535` both hard and soft types.
+
+            * soft nofile 65535
+            * hard nofile 65535
+
+    -   **rss**
+
+        Maximum resident set size in KB (ignored in Linux 2.4.30 and higher), we recommend to change to `unlimited` both
+        hard and soft types.
+
+            * soft rss unlimited
+            * hard rss unlimited
+
+    -   **stack**
+
+        Maximum stack size in KB, we recommend to change to `unlimited` both hard and soft types.
+
+            * soft stack unlimited
+            * hard stack unlimited
+
+    -   **nproc**
+
+        Maximum number of processes, we recommend to change to `unlimited` both hard and soft types.
+
+            * soft nproc unlimited
+            * hard nproc unlimited
+
+    You can take a look to the [limits.conf](limits.conf) file provided in this folder with all the values provided.
+
+-   **Configure kernel parameters**
+
+    sysctl is used to modify kernel parameters at runtime. We plan to modify the corresponding `/etc/sysctl.conf` file.
+    You can get more information in the corresponding man pages of
+    [sysctl](http://man7.org/linux/man-pages/man8/sysctl.8.html) and
+    [sysctl.conf](http://man7.org/linux/man-pages/man5/sysctl.conf.5.html). You can search all the kernel parameters by
+    using the command `sysctl -a`
+
+    -   **fs.file-max**
+
+        The maximum file handles that can be allocated, the recommended value is `1000000`.
+
+            fs.file-max = 1000000
+
+    -   **fs.nr_open**
+
+        Max amount of file handles that can be opened, the recommended value is `1000000`.
+
+            fs.nr_open = 1000000
+
+    -   **net.netfilter.nf_conntrack_max**
+
+        Size of connection tracking table. Default value is nf_conntrack_buckets value \* 4.
+
+            net.nf_conntrack_max = 1048576
+
+    For more details about any other kernel parameters, take a look to the example [sysctl.conf](sysctl.conf) file.

docs/limits.conf

@@ -0,0 +1,33 @@
+### USER LIMITS TUNING ###
+
+# Maximum number of open file descriptors
+* soft nofile 65535
+* hard nofile 65535
+
+# Maximum number of processes
+* soft nproc unlimited
+* hard nproc unlimited
+
+# Maximum filesize in KB
+* soft fsize unlimited
+* hard fsize unlimited
+
+# Maximum data size in KB
+* soft data unlimited
+* hard data unlimited
+
+# Maximum stack size in KB
+* soft stack unlimited
+* hard stack unlimited
+
+# Maximum resident set size in KB (ignored in Linux 2.4.30 and higher)
+* soft rss unlimited
+* hard rss unlimited
+
+# Limits of the core file size in KB
+* soft core unlimited
+* hard core unlimited
+
+# Maximum locked-in-memory address space in KB
+* memlock unlimited
+* memlock unlimited