picluster/server/server.js
Verita84 48111c89c0 fix
2021-08-25 08:44:55 -06:00

2145 lines
68 KiB
JavaScript

const http = require('http');
const https = require('https');
const fs = require('fs');
const net = require('net');
const tls = require('tls');
const multer = require('multer');
const express = require('express');
const Moment = require('moment');
const async = require('async');
const superagent = require('superagent');
const bodyParser = require('body-parser');
const {
forOwn
} = require('lodash');
const {
parse
} = require('path');
const {
exec
} = require('child_process');
const version = "3.0.11"
const bootstrap = {
status: 1
};
const functions = {
name: []
};
let total_nodes = 0;
let config;
let config_file;
if (process.env.PICLUSTER_CONFIG) {
config = JSON.parse(fs.readFileSync(process.env.PICLUSTER_CONFIG, 'utf8'));
config_file = process.env.PICLUSTER_CONFIG;
} else {
config = JSON.parse(fs.readFileSync('../config.json', 'utf8'));
config_file = '../config.json';
}
if (config.ssl_self_signed) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
}
const app = express();
app.use(express.json({
limit: '20mb'
}));
const upload = multer({
dest: '../'
});
const scheme = config.ssl ? 'https://' : 'http://';
const ssl_self_signed = config.ssl_self_signed === false;
const server = config.web_connect;
let {
rsyslog_host
} = config;
const {
server_port
} = config;
const {
agent_port
} = config;
let log = '';
let {
token
} = config;
let dockerFolder = config.docker;
const container_faillog = [];
const picluster_release = '2.6';
if (config.loadbalancer) {
configure_loadbalancer();
}
if (config.elasticsearch) {
const mapping = {
settings: {
index: {
number_of_shards: 3,
number_of_replicas: 2
}
},
mappings: {
'picluster-logging': {
properties: {
date: {
type: 'date',
index: 'true',
},
data: {
type: 'keyword',
index: 'true'
}
}
}
}
};
const monitoring_mapping = {
settings: {
index: {
number_of_shards: 3,
number_of_replicas: 2
}
},
mappings: {
'picluster-monitoring': {
properties: {
date: {
type: 'date',
index: 'true'
},
cpu: {
type: 'double',
index: 'true'
},
node: {
type: 'text',
index: 'true'
},
memory: {
type: 'double',
index: 'true'
},
network_tx: {
type: 'double',
index: 'true'
},
network_rx: {
type: 'double',
index: 'true'
},
disk: {
type: 'double',
index: 'true'
},
total_running_containers: {
type: 'double',
index: 'true'
}
}
}
}
};
create_es_mappings(mapping, 'picluster-logging');
create_es_mappings(monitoring_mapping, 'picluster-monitoring');
}
//templates
/**
superagent
.get(`${scheme}${node}:${agent_port}/node-status`)
.query({ token: token })
.end((error, response) => {
});
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({ token: token, payload })
.set('accept', 'json')
.end((error, response) => {
});
*/
function create_es_mappings(mapping, index) {
superagent
.put(config.elasticsearch + '/' + index)
.send(mapping)
.set('accept', 'json')
// .set('Authorization', 'ApiKey Ui1YYTFYb0JvcUN3M1lvSlBvX0E6Rl9PamI2aU1RM1NvV2xwTHQ0bFpuQQ')
.end((error, response) => {
console.log('\nCreating Elasticsearch Map......');
if (error) {
console.log(error);
} else {
console.log(response.text);
}
});
}
if (config.automatic_heartbeat) {
if (config.automatic_heartbeat.indexOf('enabled') > -1) {
if (config.heartbeat_interval) {
console.log('\nAutomatic Heartbeat Enabled. Will check every: ' + config.heartbeat_interval + ' ms.');
automatic_heartbeat();
} else {
console.log('\nAutomatic Heartbeat Disabled: heartbeat_interval is not set.');
}
} else {
console.log('\nAutomatic Heartbeat Disabled.');
}
} else {
console.log('\nAutomatic Heartbeat Disabled.');
}
function automatic_heartbeat() {
if (config.automatic_heartbeat.indexOf('enabled') > -1) {
setTimeout(() => {
superagent
.get(`${scheme}${server}:${server_port}/hb`)
.query({
token: token
})
.end((error, response) => {
if (error) {
console.log(error);
}
});
automatic_heartbeat();
}, config.heartbeat_interval);
} else {
console.log('\nAutomatic Heartbeat Disabled.');
}
}
app.get('/clear-functions', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
Object.keys(functions.name).forEach((get_name, i) => {
delete_function(functions.name[i].name, functions.name[i].host);
remove_function_data(functions.name[i].uuid);
});
res.end('Sent request to remove stale functions.');
}
});
app.post('/bootstrap', (req, res) => {
const check_token = req.body.token;
const {
host
} = req.body;
let statusCode = 0;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials or missing parameters.');
} else {
if (bootstrap.status === 1) {
let proceed = 1;
Object.keys(config.layout).forEach((get_node, i) => {
if (config.layout[i].node.indexOf(host) > -1) {
proceed = 0;
}
});
if (proceed) {
config.layout.push({
node: host
});
config.hb.push({
node: host
});
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end('An error occurred: ' + error);
} else {
bootstrap.status = 1;
console.log('\nAdded node: ' + host + ' to the cluster.');
statusCode = 1;
}
});
} else {
bootstrap.status = 1;
console.log('\nnode: ' + host + ' is already part of the cluster.');
statusCode = 2;
}
} else {
console.log('\nAnother bootstrap process is already running. Please try again later.');
statusCode = 0;
}
res.end(JSON.stringify({
output: statusCode
}));
}
});
app.post('/function', (req, res) => {
const check_token = req.body.token;
const {
output
} = req.body;
const {
uuid
} = req.body;
if ((check_token !== token) || (!check_token) || (!uuid)) {
res.end('\nError: Invalid Credentials or missing parameters.');
} else {
Object.keys(functions.name).forEach((get_name, i) => {
if (functions.name[i].uuid.toString().indexOf(uuid.toString()) > -1) {
functions.name[i].output = output;
delete_function(functions.name[i].name, functions.name[i].host);
res.end('');
}
});
}
});
app.get('/function', (req, res) => {
const check_token = req.query.token;
const name = req.query.function;
const min = 1;
const max = 9999999;
const uuid = Math.floor(Math.random() * (max - min + 1)) + min;
const min_node = 0;
const max_node = total_nodes - 1;
const node_number = Math.floor(Math.random() * (max_node - min_node + 1)) + min_node;
const host = config.layout[node_number].node;
const {
container_args
} = req.query;
const function_data = {
uuid,
name: name + '-' + uuid,
output: '',
host
};
if ((check_token !== token) || (!check_token) || (!name)) {
res.end('\nError: Invalid Credentials or parameters.');
} else {
functions.name.push(function_data);
create_function(name + '-' + uuid, uuid, host, container_args);
res.end(scheme + server + ':' + server_port + '/getfunction?token=' + token + '&uuid=' + uuid);
}
});
function remove_function_data(uuid) {
Object.keys(functions.name).forEach((get_name, i) => {
if (functions.name[i].uuid.toString().indexOf(uuid.toString()) > -1) {
functions.name[i].name = '';
functions.name[i].output = '';
functions.name[i].uuid = '';
functions.name[i].host = '';
}
});
}
app.get('/getfunction', (req, res) => {
const check_token = req.query.token;
const {
uuid
} = req.query;
let output = '';
if ((check_token !== token) || (!check_token) || (!uuid)) {
res.end('\nError: Invalid Credentials or parameters.');
} else {
Object.keys(functions.name).forEach((get_name, i) => {
if ((functions.name[i].uuid.toString().indexOf(uuid.toString()) > -1 && functions.name[i].output.length > 1)) {
output = functions.name[i].output;
remove_function_data(uuid);
}
});
res.end(output);
}
});
function create_function(name, uuid, host, user_container_args) {
let container_args = '-e UUID=' + uuid + ' -e TOKEN=' + token + ' -e SERVER=' + scheme + server + ':' + server_port;
const container = name;
if (user_container_args) {
container_args = user_container_args + ' ' + container_args;
}
migrate(container, host, host, container_args, uuid);
}
app.get('/clearlog', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
log = '';
res.end();
}
});
app.get('/nodes', (req, res) => {
const node_metrics = {
version,
data: [],
functions,
function_server: `${scheme}${server}:${server_port}/getfunction`
};
function addData(data) {
node_metrics.data.push(data);
}
function getData() {
let total_node_count = 0;
let total_containers = 0;
const node_list = [];
const container_list = [];
for (let i = 0; i < config.layout.length; i++) {
for (const key in config.layout[i]) {
if (config.layout[i].hasOwnProperty(key)) {
const {
node
} = config.layout[i];
const node_info = config.layout[i][key];
if (node_info === node) {
total_node_count++;
node_list.push(node);
} else {
total_containers++;
container_list.push(key);
}
}
}
}
node_metrics.total_containers = total_containers;
node_metrics.total_nodes = total_node_count;
node_metrics.container_list = container_list;
node_metrics.nodes = node_list;
total_nodes = total_node_count;
return node_metrics;
}
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
config.layout.forEach(get_node => {
const {
node
} = get_node;
if (!node) {
console.error('Invalid Config for node', get_node);
return;
}
superagent
.get(`${scheme}${node}:${agent_port}/node-status`)
.query({
token: token
})
.end((error, response) => {
if (error) {
console.error(error);
} else {
try {
const check = JSON.parse(response.text);
if (check.cpu_percent > 0) {
addData(check);
if (config.elasticsearch) {
elasticsearch_monitoring(check.cpu_percent / check.cpu_cores, check.hostname, check.disk_percentage, check.memory_percentage, check.total_running_containers, check.network_rx, check.network_tx);
}
}
} catch (error2) {
console.log('\nError gathering monitoring metrics: Invalid JSON or Credentials!' + error2);
}
}
});
});
setTimeout(() => {
res.json(getData());
}, 3000);
}
});
function addLog(data) {
log += data;
if (config.elasticsearch) {
elasticsearch(JSON.stringify(log));
}
}
app.get('/', (req, res) => {
res.end('PiCluster Server v' + picluster_release);
});
app.get('/manage-image', (req, res) => {
const check_token = req.query.token;
const {
operation
} = req.query;
let docker_command = '';
let container = '';
let command_log = '';
const url = [];
const what = [];
const {
no_cache
} = req.query;
if (req.query.container) {
container = req.query.container;
}
if (container.indexOf('*') > -1 || container.length === 0) {
container = '*';
}
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
Object.keys(config.layout).forEach((get_node, i) => {
Object.keys(config.layout[i]).forEach(key => {
const {
node
} = config.layout[i];
if ((!config.layout[i].hasOwnProperty(key) || key.indexOf('node') > -1)) {
return;
}
const make_url = `${scheme}${node}:${agent_port}/run`;
if (container.indexOf('*') > -1 || container.indexOf(key) > -1) {
what.push(key);
url.push(make_url);
}
});
});
let i = 0;
async.eachSeries(url, (url, cb) => {
if (operation === 'rm') {
docker_command = 'docker image rm ' + what[i];
}
if (operation === 'build' && no_cache === '1') {
docker_command = 'docker image build --no-cache ' + dockerFolder + '/' + what[i] + ' -t ' + what[i] + ' -f ' + dockerFolder + '/' + what[i] + '/Dockerfile';
}
if (operation === 'build' && no_cache === '0') {
docker_command = 'docker image build ' + dockerFolder + '/' + what[i] + ' -t ' + what[i] + ' -f ' + dockerFolder + '/' + what[i] + '/Dockerfile';
}
superagent
.post(url)
.send({
token: token,
command: docker_command
})
.set('accept', 'json')
.end((error, response) => {
try {
const data = JSON.parse(response.text);
command_log += 'Node: ' + data.node + '\n\n' + data.output + '\n\n';
cb(error);
} catch (error) {
console.log(error);
}
});
i++;
}, err => {
if (err) {
console.log('\nError: ' + err);
}
res.end(command_log);
});
}
});
app.get('/manage', (req, res) => {
const check_token = req.query.token;
const {
operation
} = req.query;
let docker_command = '';
let command_log = '';
let container = '';
const url = [];
const what = [];
const args = [];
if (req.query.container) {
container = req.query.container;
}
if (operation === 'start') {
docker_command = 'docker container start';
}
if (operation === 'stop') {
docker_command = 'docker container stop';
}
if (operation === 'rm') {
docker_command = 'docker container rm -f';
}
if (operation === 'restart') {
docker_command = 'docker container restart';
}
if (operation === 'logs') {
docker_command = 'docker container logs';
}
if (operation === 'create') {
docker_command = 'docker container run -d --name ';
}
if (operation === 'commit' && req.query.container) {
docker_command = 'su root -s /bin/bash -lc "docker commit $(docker ps | grep -i ' + req.query.container + ' | cut -d \" \" -f1 | tail -1) ' + req.query.container + "\"";
}
if (container.indexOf('*') > -1 || container.length === 0) {
container = '*';
}
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
Object.keys(config.layout).forEach((get_node, i) => {
Object.keys(config.layout[i]).forEach(key => {
const {
node
} = config.layout[i];
if ((!config.layout[i].hasOwnProperty(key) || key.indexOf('node') > -1)) {
return;
}
const make_url = `${scheme}${node}:${agent_port}/run`;
if (container.indexOf('*') > -1 || container.indexOf(key) > -1) {
what.push(key);
url.push(make_url);
args.push(config.layout[i][key]);
}
});
});
let i = 0;
async.eachSeries(url, (url, cb) => {
let command;
if (operation === 'create') {
command = docker_command + what[i] + ' ' + args[i] + ' ' + what[i];
} else {
command = docker_command + ' ' + what[i];
}
superagent
.post(url)
.send({
token: token,
command: command
})
.set('accept', 'json')
.end((error, response) => {
try {
const data = JSON.parse(response.text);
command_log += 'Node: ' + data.node + '\n\n' + data.output + '\n\n';
cb(error);
} catch (error) {
console.log(error);
}
});
i++;
}, err => {
if (err) {
console.log('\nError: ' + err);
}
res.end(command_log);
});
}
});
function migrate(container, original_host, new_host, original_container_data, uuid) {
let existing_automatic_heartbeat_value = '';
if (config.automatic_heartbeat) {
existing_automatic_heartbeat_value = config.automatic_heartbeat;
if (config.automatic_heartbeat.indexOf('enabled') > -1) {
config.automatic_heartbeat = 'disabled';
}
}
superagent
.post(`${scheme}${original_host}:${agent_port}/run`)
.send({
token: token,
command: 'docker container rm -f ' + container
})
.set('accept', 'json')
.end((error, response) => {
try {
let command = '';
if (uuid) {
const image_name = container.split('-' + uuid)[0];
command = 'docker image build ' + dockerFolder + '/' + image_name + ' -t ' + image_name + ' -f ' + dockerFolder + '/' + image_name + '/Dockerfile;docker container run -d --name ' + container + ' ' + original_container_data + ' ' + image_name;
} else {
command = 'docker image build ' + dockerFolder + '/' + container + ' -t ' + container + ' -f ' + dockerFolder + '/' + container + '/Dockerfile;docker container run -d --name ' + container + ' ' + original_container_data + ' ' + container;
}
superagent
.post(`${scheme}${new_host}:${agent_port}/run`)
.send({
token: token,
command: command
})
.set('accept', 'json')
.end((second_error, second_response) => {
try {
if (config.automatic_heartbeat) {
if (existing_automatic_heartbeat_value.indexOf('enabled') > -1) {
config.automatic_heartbeat = existing_automatic_heartbeat_value;
}
}
} catch (second_error) {
addLog('An error has occurred');
console.log('\nError with migration: ' + second_error);
}
});
} catch (error) {
addLog('An error has occurred with migration:');
console.log(error);
}
});
}
app.get('/addhost', (req, res) => {
const check_token = req.query.token;
const {
host
} = req.query;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
let proceed = 1;
for (let i = 0; i < config.layout.length; i++) {
if (config.layout[i].node.indexOf(host) > -1) {
proceed = 0;
}
}
if (proceed) {
config.layout.push({
node: host
});
config.hb.push({
node: host
});
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
res.end('\nAdded host ' + host + ' to the configuration.');
}
});
} else {
res.end('\nError: Host already exists');
}
}
});
function elasticsearch_monitoring(cpu, node, disk, memory, total_running_containers, network_rx, network_tx) {
const current_time = new Moment();
superagent
.post(config.elasticsearch + '/picluster-monitoring/picluster-monitoring')
.send({
date: current_time,
cpu,
node,
disk,
memory,
network_rx,
network_tx,
total_running_containers
})
.set('accept', 'json')
//.set('Authorization', 'ApiKey Ui1YYTFYb0JvcUN3M1lvSlBvX0E6Rl9PamI2aU1RM1NvV2xwTHQ0bFpuQQ')
.end((error, response) => {
if (error) {
console.log(error);
}
});
}
function elasticsearch(data) {
const current_time = new Moment();
superagent
.post(config.elasticsearch + '/picluster-logging/picluster-logging')
.send({
date: current_time,
data
})
.set('accept', 'json')
// .set('Authorization', 'ApiKey Ui1YYTFYb0JvcUN3M1lvSlBvX0E6Rl9PamI2aU1RM1NvV2xwTHQ0bFpuQQ')
.end((error, response) => {
if (error) {
console.log(error);
}
});
}
app.get('/rmhost', (req, res) => {
const check_token = req.query.token;
const {
host
} = req.query;
let hb_proceed = 0;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
// Ensures that the host exists
for (let i = 0; i < config.layout.length; i++) {
if (config.layout[i].node.indexOf(host) > -1) {
config.layout.splice(i, 1);
hb_proceed = 1;
break;
}
}
}
if (hb_proceed) {
if (config.hb) {
for (let i = 0; i < config.hb.length; i++) {
if (config.hb[i].node.indexOf(host) > -1) {
config.hb.splice(i, 1);
break;
}
}
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
res.end('\nRemoved host ' + host + ' from the configuration.');
}
});
});
app.get('/removecontainerconfig', (req, res) => {
const check_token = req.query.token;
const {
container
} = req.query;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
Object.keys(config.layout).forEach((get_node, i) => {
Object.keys(config.layout[i]).forEach(key => {
if ((!config.layout[i].hasOwnProperty(key) || key.indexOf('node') > -1)) {
return;
}
if (key.indexOf(container) > -1) {
delete config.layout[i][key];
}
});
});
if (config.hb) {
Object.keys(config.hb).forEach((get_node, i) => {
Object.keys(config.hb[i]).forEach(key => {
if ((!config.hb[i].hasOwnProperty(key) || key.indexOf('node') > -1)) {
return;
}
if (key.indexOf(container) > -1) {
delete config.hb[i][key];
}
});
});
}
if (config.container_host_constraints) {
Object.keys(config.container_host_constraints).forEach((get_node, i) => {
Object.keys(config.container_host_constraints[i]).forEach(key => {
const analyze = config.container_host_constraints[i][key].split(',');
if (container.indexOf(analyze[0]) > -1) {
config.container_host_constraints.splice(i, i + 1);
}
});
});
for (let i = 0; i < config.container_host_constraints.length; i++) {
for (const key in config.container_host_constraints[i]) {
if (container.length > 0) {
const analyze = config.container_host_constraints[i][key].split(',');
if (container.indexOf(analyze[0]) > -1) {
config.container_host_constraints.splice(i, i + 1);
}
}
}
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
res.end('\nRemoved Container ' + container + ' from the configuration.');
}
});
}
});
app.get('/addcontainer', (req, res) => {
const check_token = req.query.token;
let {
host
} = req.query;
const {
container
} = req.query;
const {
container_args
} = req.query;
const {
heartbeat_args
} = req.query;
const {
failover_constraints
} = req.query;
if (host.indexOf('*') > -1) {
const min = 0;
const max = total_nodes - 1;
const number = Math.floor(Math.random() * (max - min + 1)) + min;
host = config.layout[number].node;
}
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
// Ensures that the host exists
let proceed = 0;
for (let i = 0; i < config.layout.length; i++) {
if (config.layout[i].node.indexOf(host) > -1) {
proceed++;
}
}
if (proceed < 1) {
res.end('\nError: Node does not exist!');
} else {
// Add Data to New Host
for (let i = 0; i < config.layout.length; i++) {
if (config.layout[i].node.indexOf(host) > -1) {
config.layout[i][container] = container_args;
}
}
// Adds Heartbeat Data
if (config.hb) {
if (heartbeat_args) {
for (let i = 0; i < config.hb.length; i++) {
if (config.hb[i].node.indexOf(host) > -1) {
config.hb[i][container] = heartbeat_args;
}
}
}
}
if (config.container_host_constraints) {
if (failover_constraints) {
config.container_host_constraints.push({
container: failover_constraints
});
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
superagent
.get(`${scheme}${server}:${server_port}/changehost`)
.query({
token: token,
container,
newhost: host,
})
.end((error, response) => {
if (!error && response.text) {
res.end('\nAdded ' + container + ' to the configuration.');
} else {
res.end('\nError connecting with server.');
}
});
}
});
}
}
});
function configure_loadbalancer() {
let nginx_configuration = 'stream { \n ';
Object.keys(config.loadbalancer).forEach((get_node, i) => {
Object.keys(config.loadbalancer[i]).forEach(key => {
const parse_data = config.loadbalancer[i][key].split(',');
const container_name = parse_data[0];
const container_port = parse_data[parse_data.length - 2];
const service_port = parse_data[parse_data.length - 1];
const lb_hosts = parse_data.toString().split(',');
nginx_configuration += '\n\n upstream ' + container_name + ' {';
for (let i = 1; i < lb_hosts.length - 2; i++) {
nginx_configuration += '\nserver ' + lb_hosts[i] + ':' + container_port + ';';
}
nginx_configuration += '\n } \n\nserver { \n listen ' + ' ' + service_port + '; \n proxy_pass ' + container_name + ';\n }';
});
});
nginx_configuration += '\n }';
console.log(nginx_configuration);
fs.writeFile('picluster.conf', nginx_configuration, err => {
if (err) {
console.log(err);
} else {
command = 'docker container stop picluster_lb;docker cp picluster.conf picluster_lb:/etc/nginx/conf.d/;docker container restart picluster_lb ';
exec(command, (error, stdout, stderr) => {
if (error) {
console.log(stderr);
} else {
console.log(stdout);
}
}, err => {
if (err) {
console.error('error:', err);
}
});
}
});
}
app.get('/update-container', (req, res) => {
const check_token = req.query.token;
const {
container
} = req.query;
const {
container_args
} = req.query;
const {
heartbeat_args
} = req.query;
const {
failover_constraints
} = req.query;
if ((check_token !== token) || (!check_token) || container.indexOf('*') > -1) {
res.end('\nError: Invalid Credentials');
} else {
if (container_args) {
Object.keys(config.layout).forEach((get_node, i) => {
Object.keys(config.layout[i]).forEach(key => {
if (key.indexOf(container) > -1) {
config.layout[i][key] = container_args;
}
});
});
}
if (failover_constraints) {
let proceed = 0;
Object.keys(config.container_host_constraints).forEach((get_node, i) => {
Object.keys(config.container_host_constraints[i]).forEach(key => {
const get_container_name = failover_constraints.split(',');
const parse_container = get_container_name[0];
if (config.container_host_constraints[i][key].indexOf(parse_container) > -1) {
if (failover_constraints.indexOf('none') > -1) {
proceed = 0;
} else {
proceed = 1;
config.container_host_constraints[i][key] = failover_constraints;
}
}
});
});
if (proceed === 0) {
if (failover_constraints.indexOf('none') > -1) {
for (let i = 0; i < config.container_host_constraints.length; i++) {
for (const key in config.container_host_constraints[i]) {
if (container.length > 0) {
const analyze = config.container_host_constraints[i][key].split(',');
if (container.indexOf(analyze[0]) > -1) {
config.container_host_constraints.splice(i, i + 1);
}
}
}
}
} else {
config.container_host_constraints.push({
container: failover_constraints
});
}
}
}
if (heartbeat_args) {
let proceed = 0;
if (config.hb.length === 0) {
for (let i = 0; i < config.layout.length; i++) {
config.hb.push({
node: config.layout[i].node
});
}
}
Object.keys(config.hb).forEach((get_node, i) => {
Object.keys(config.hb[i]).forEach(key => {
if (key.indexOf(container) > -1) {
if (heartbeat_args.indexOf('delete') > -1) {
delete config.hb[i][key];
proceed = 1;
} else {
config.hb[i][key] = heartbeat_args;
proceed = 1;
}
}
});
});
if (proceed === 0) {
let node = '';
Object.keys(config.layout).forEach((get_node, i) => {
Object.keys(config.layout[i]).forEach(key => {
if (key.indexOf(container) > -1) {
node = config.layout[i].node;
}
});
});
for (let i = 0; i < config.hb.length; i++) {
if (config.hb[i].node.indexOf(node) > -1 && heartbeat_args.indexOf('delete') === -1) {
config.hb[i][container] = heartbeat_args;
}
}
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
res.end('\nModified Container Arguments for ' + container);
}
});
}
});
app.get('/lb', (req, res) => {
const check_token = req.query.token;
const {
container
} = req.query;
const {
container_port
} = req.query;
const {
service_port
} = req.query;
const {
lb_hosts
} = req.query;
if ((check_token !== token) || (!check_token) || container.indexOf('*') > -1) {
res.end('\nError: Invalid Credentials');
} else {
if (lb_hosts) {
let proceed = 0;
Object.keys(config.loadbalancer).forEach((get_node, i) => {
Object.keys(config.loadbalancer[i]).forEach(key => {
const get_container_name = lb_hosts.split(',');
const parse_container = get_container_name[0];
if (config.loadbalancer[i][key].indexOf(parse_container) > -1) {
if (lb_hosts.indexOf('none') > -1) {
proceed = 0;
} else {
proceed = 1;
config.loadbalancer[i][key] = lb_hosts + ',' + container_port + ',' + service_port;
}
}
});
});
if (proceed === 0) {
if (lb_hosts.indexOf('none') > -1) {
for (let i = 0; i < config.loadbalancer.length; i++) {
for (const key in config.loadbalancer[i]) {
if (container.length > 0) {
const analyze = config.loadbalancer[i][key].split(',');
if (container.indexOf(analyze[0]) > -1) {
config.loadbalancer.splice(i, i + 1);
}
}
}
}
} else {
config.loadbalancer.push({
container: lb_hosts + ',' + container_port + ',' + service_port
});
}
}
configure_loadbalancer();
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
res.end('\nModified Container Arguments for ' + container);
}
});
}
});
app.get('/changehost', (req, res) => {
const check_token = req.query.token;
let container = '';
let original_host = '';
let original_container_data = '';
let original_heartbeat_data = '';
const new_host = req.query.newhost;
if (req.query.container) {
container = req.query.container;
}
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
// Ensures that the host exists
let proceed = 0;
for (let i = 0; i < config.layout.length; i++) {
for (const key in config.layout[i]) {
if (container.length > 0) {
if (config.layout[i].node.indexOf(new_host) > -1) {
proceed++;
}
if (key.indexOf(container) > -1) {
if (key.indexOf(config.layout[i].node)) {
proceed++;
}
}
}
}
}
// Find Current Host
if (proceed < 2) {
res.end('\nError: Node or Container does not exist!');
} else {
for (let i = 0; i < config.layout.length; i++) {
for (const key in config.layout[i]) {
if (container.length > 0) {
if (key.indexOf(container) > -1) {
original_host = config.layout[i].node;
original_container_data = config.layout[i][key];
delete config.layout[i][key];
}
}
}
}
// Checks for HB
if (config.hb) {
for (let i = 0; i < config.hb.length; i++) {
for (const key in config.hb[i]) {
if (container.length > 0) {
if (key.indexOf(container) > -1) {
original_heartbeat_data = config.hb[i][key];
delete config.hb[i][key];
}
}
}
}
}
for (let i = 0; i < config.layout.length; i++) {
if (config.layout[i].node.indexOf(new_host) > -1) {
config.layout[i][container] = original_container_data;
}
}
// Adds Heartbeat Data
if (config.hb) {
if (original_heartbeat_data) {
for (let i = 0; i < config.hb.length; i++) {
if (config.hb[i].node.indexOf(new_host) > -1) {
config.hb[i][container] = original_heartbeat_data;
}
}
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end(error);
} else {
migrate(container, original_host, new_host, original_container_data);
res.end('\nMigration may take awhile. Please observe the logs and running containers for the latest information.');
}
});
}
}
});
function delete_function(name, node) {
superagent
.post(scheme + node + ':' + agent_port + '/run')
.send({
token: token,
command: 'docker container rm -f ' + name
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
console.log('\n' + error);
}
});
}
app.post('/listcontainers', (req, res) => {
let {
node
} = req.body;
const check_token = req.body.token;
const output = [];
let container;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
for (const key in config.layout[i]) {
if (config.layout[i].hasOwnProperty(key)) {
container = key;
node = config.layout[i].node;
const check_port = config.layout[i][key];
if (check_port !== node) {
output.push(container);
}
}
}
}
res.send(output);
}
});
app.post('/listnodes', (req, res) => {
const check_token = req.body.token;
const output = [];
let node;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
for (const key in config.layout[i]) {
if (config.layout[i].hasOwnProperty(key)) {
node = config.layout[i].node;
const port_check = config.layout[i][key];
if (port_check === node) {
output.push(node);
}
}
}
}
res.send(output);
}
});
function copyToAgents(data, file, config_file, temp_file) {
Object.keys(config.layout).forEach((get_node, i) => {
const {
node
} = config.layout[i];
const formData = {
name: file,
token,
config_file: config_file,
data: data
};
setTimeout(() => {
superagent
.post(`${scheme}${node}:${agent_port}/receive-file`)
.send(formData)
.set('accept', 'json')
.end((error, response) => {
try {
if (!config_file) {
addLog('\nCopied ' + file + ' to ' + node);
console.log('\nCopied ' + file + ' to ' + node);
}
} catch (error) {
console.log('\nResponse= ' + response);
console.log('\n' + formData);
console.log('\nError sending file to agent: ' + error);
}
});
}, 3000);
});
if (temp_file) {
fs.unlink(temp_file, error => {
if (error) {
console.log(error);
}
});
}
}
app.post('/receive-file', upload.single('file'), (req, res) => {
const check_token = req.body.formData.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
copyToAgents(req.body, req.body.formData.original_name, '', req.body.formData.file.path);
res.end('');
}
});
app.post('/listcommands', (req, res) => {
const check_token = req.body.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else if (config.commandlist) {
res.end(JSON.stringify(config.commandlist));
} else {
res.end('');
}
});
function swarm_remove() {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
superagent
.post(`${scheme}${node}:${agent_port}/run`)
.send({
token: token,
command: 'docker swarm leave --force'
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
console.log('An error has occurred.');
} else {
const results = JSON.parse(response.text);
addLog('\nNode:' + results.node + '\n' + results.output);
}
});
}
}
function swarm_nodes(swarm_token, host) {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
if (host.indexOf(node) > -1) {
console.log('\n' + node + ' is already set as the master.');
} else {
superagent
// .post(`${scheme}${server}:${server_port}/updateconfig`
.post(`${scheme}${node}:${agent_port}/run`)
.send({
token: token,
command: 'docker swarm join --token ' + swarm_token + ' ' + host
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
console.log('An error has occurred.');
} else {
const results = JSON.parse(response.text);
console.log(results);
addLog('\nNode:' + results.node + '\n' + results.output);
}
});
}
}
}
app.post('/swarm-create', (req, res) => {
const check_token = req.body.token;
const {
host
} = req.body;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
if (host.indexOf(node) > -1) {
superagent
.post(`${scheme}${node}:${agent_port}/run`)
.send({
token: token,
command: 'docker swarm init'
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end('An error has occurred.');
} else {
const results = JSON.parse(response.text);
const get_output = results.output.toString();
if (get_output.indexOf('SWMTKN') > -1 || config.swarm_token) {
if (!config.swarm_token) {
const get_swarm_token_line = get_output.split('--token');
const get_swarm_token = get_swarm_token_line[1].split(' ');
config.swarm_token = get_swarm_token[1];
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end('An error occurred: ' + error);
} else {
bootstrap.status = 1;
console.log('\nAdded Swarm Token to config file.');
}
});
}
swarm_nodes(config.swarm_token, host);
} else {
res.end('Error creating Swarm.' + results.output);
}
}
});
}
}
res.end('Swarm Operation Complete');
}
});
app.post('/swarm-network-create', (req, res) => {
const check_token = req.body.token;
const {
host
} = req.body;
const {
network
} = req.body;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
if (host.indexOf(node) > -1) {
superagent
.post(`${scheme}${node}:${agent_port}/run`)
.send({
token: token,
command: 'docker network create -d overlay --attachable ' + network
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end('An error has occurred.');
} else {
const results = JSON.parse(response.text);
res.end(results.output);
}
});
}
}
res.end('');
}
});
app.post('/swarm-remove', (req, res) => {
if (config.swarm_token) {
delete config.swarm_token;
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
console.log('An error occurred: ' + error);
} else {
bootstrap.status = 1;
console.log('\nRemoved Swarm Token from config file.');
}
});
}
const check_token = req.body.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
swarm_remove();
res.end('\nRemoved Swarm Token from config file.');
}
});
app.post('/exec', (req, res) => {
const check_token = req.body.token;
let selected_node = '';
let command_log = '';
const url = [];
if (req.body.node) {
selected_node = req.body.node;
}
if (selected_node.indexOf('*') > -1) {
selected_node = '';
}
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
const make_url = `${scheme}${node}:${agent_port}/run`;
if (selected_node.length > -1 && selected_node.indexOf(node) > -1) {
url.push(make_url);
}
if (selected_node.length === 0) {
url.push(make_url);
}
}
async.eachSeries(url, (url, cb) => {
superagent
.post(url)
.send({
token: token,
command: req.body.command
})
.set('accept', 'json')
.end((error, response) => {
try {
if ((response && response.text)) {
const data = JSON.parse(response.text);
command_log += 'Node: ' + data.node + '\n\n' + data.output + '\n\n';
}
cb(error);
} catch (error) {
console.log(error);
}
});
}, err => {
if (err) {
console.log('\nError: ' + err);
}
res.end(command_log);
});
}
});
app.post('/syslog', (req, res) => {
const check_token = req.body.token;
let complete_syslog = '';
const url = [];
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
make_url = `${scheme}${node}:${agent_port}/run`;
url.push(make_url);
}
async.eachSeries(url, (url, cb) => {
superagent
.post(url)
.send({
token: token,
command: config.syslog
})
.set('accept', 'json')
.end((error, response) => {
try {
const data = JSON.parse(response.text);
complete_syslog += 'Node: ' + data.node + '\n\n' + data.output + '\n\n';
cb(error);
} catch (error) {
console.log(error);
}
});
}, err => {
if (err) {
console.log('\nError: ' + err);
}
res.end(complete_syslog);
});
}
});
app.post('/prune', (req, res) => {
const check_token = req.body.token;
const url = [];
let command_log = '';
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
for (let i = 0; i < config.layout.length; i++) {
const {
node
} = config.layout[i];
const make_url = `${scheme}${node}:${agent_port}/run`;
url.push(make_url);
}
async.eachSeries(url, (url, cb) => {
superagent
.post(url)
.send({
token: token,
command: 'docker system prune -a -f'
})
.set('accept', 'json')
.end((error, response) => {
try {
const data = JSON.parse(response.text);
command_log += 'Node: ' + data.node + '\n\n' + data.output + '\n\n';
cb(error);
} catch (error) {
console.log(error);
}
});
}, err => {
if (err) {
console.log('\nError: ' + err);
}
res.end(command_log);
});
}
});
function move_container(container, newhost) {
console.log('\nMigrating container ' + container + ' to ' + newhost + '......');
addLog('\nMigrating container ' + container + ' to ' + newhost + '......');
superagent
.get(`${scheme}${server}:${server_port}/changehost`)
.query({
token: token,
container,
newhost
})
.end((error, response) => {
if (error) {
console.log('Error connecting with server. ' + error);
} else {
config.automatic_heartbeat = 'enabled';
}
});
}
function container_failover(container) {
let container_fail_counter = 0;
let proceed = '';
for (const key in container_faillog) {
if (log.hasOwnProperty(key)) {
if (container_faillog[key].indexOf(container) > -1) {
container_fail_counter++;
}
}
}
if (container_fail_counter >= 3) {
for (const bkey in container_faillog) {
if (container_faillog[bkey].indexOf(container) > -1) {
delete container_faillog[bkey];
proceed = 1;
}
}
if (proceed) {
for (const key in config.container_host_constraints) {
if (config.container_host_constraints.hasOwnProperty(key)) {
const analyze = config.container_host_constraints[key].container.split(',');
if (container.indexOf(analyze[0]) > -1) {
analyze.splice(0, 1);
const newhost = analyze[Math.floor(Math.random() * analyze.length)];
move_container(container, newhost);
config.automatic_heartbeat = 'disabled';
}
}
}
}
}
}
function hb_check(node, container_port, container) {
if (config.automatic_heartbeat.indexOf('enabled') > -1) {
const client = config.ssl ? new tls.TLSSocket() : new net.Socket();
client.connect(container_port, node, container, () => {});
client.on('end', () => {
addLog('\nA Heart Beat Check Just Ran.');
});
client.on('error', () => {
addLog('\n' + container + ' failed on: ' + node);
console.log('\n' + container + ' failed on: ' + node);
if (config.container_host_constraints) {
container_faillog.push(container);
container_failover(container);
}
superagent
.post(`${scheme}${node}:${agent_port}/run`)
.send({
token: token,
command: 'docker restart ' + container
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
console.log(error);
}
});
});
}
}
app.get('/hb', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
let node = '';
let check_port = '';
let container = '';
for (let i = 0; i < config.hb.length; i++) {
for (const key in config.hb[i]) {
if (config.hb[i].hasOwnProperty(key)) {
container = key;
node = config.hb[i].node;
check_port = config.hb[i][key];
if (check_port !== node) {
hb_check(node, check_port, container);
}
}
}
}
res.end('');
}
});
app.get('/log', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
res.send(log);
}
});
app.get('/rsyslog', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
superagent
.get(`${scheme}${rsyslog_host}:${agent_port}/rsyslog`)
.query({
token: token
})
.end((error, response) => {
if (!error && response.text) {
res.end(response.text);
} else {
res.end('Error connecting with server. ' + error);
}
});
}
});
function reloadConfig() {
if (process.env.PICLUSTER_CONFIG) {
config = JSON.parse(fs.readFileSync(process.env.PICLUSTER_CONFIG, 'utf8'));
} else {
config = JSON.parse(fs.readFileSync('../config.json', 'utf8'));
}
token = config.token;
dockerFolder = config.docker;
rsyslog_host = config.rsyslog_host;
if (config.heartbeat_interval && config.automatic_heartbeat) {
if (config.automatic_heartbeat.indexOf('enabled') > -1) {
console.log('\nEnabing Heartbeat.');
automatic_heartbeat();
}
}
addLog('\nReloading Config.json\n');
}
app.get('/getconfig', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
res.send(config);
}
});
app.get('/gethosts', (req, res) => {
const check_token = req.query.token;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
let hosts = fs.readFileSync('/etc/hosts', 'utf8');
res.send(hosts);
}
});
app.post('/elasticsearch', (req, res) => {
const check_token = req.body.token;
const elasticsearch = req.body.elasticsearch_url;
const {
mode
} = req.body;
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
if (mode === 'add') {
if (config.elasticsearch) {
console.log('\nError, Elasticsearch is already configured.');
} else {
config.elasticsearch = elasticsearch;
console.log('\nAdded Elasticsearch configuration for: ' + elasticsearch);
}
}
if (mode === 'kibana') {
if (config.kibana) {
console.log('\nError, Kibana is already configured.');
} else {
config.kibana = elasticsearch;
console.log('\nAdded Kibana configuration for: ' + elasticsearch);
}
}
if (mode === 'delete') {
if (config.kibana) {
console.log('\nDeleted Kibana configuration.');
delete config.kibana;
}
if (config.elasticsearch) {
delete config.elasticsearch;
console.log('\nDeleted Elasticsearch configuration.');
}
}
superagent
.post(`${scheme}${server}:${server_port}/updateconfig`)
.send({
token: token,
payload: JSON.stringify(config)
})
.set('accept', 'json')
.end((error, response) => {
if (error) {
res.end('An error occurred: ' + error);
} else {
res.end();
console.log('\nUpdated Elasticsearch configuration.');
}
});
}
});
app.post('/updatehosts', (req, res) => {
let {
payload
} = req.body;
const check_token = req.body.token;
try {
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
setTimeout(() => {
// fs.writeFile(config_file, payload, err => {
// if (err) {
// console.log('\nError while writing config.' + err);
// } else {
copyToAgents(payload, '/etc/hosts', '', '');
// reloadConfig();
res.end('Updated Hosts.');
// }
// });
}, 3000);
}
} catch (error) {
res.end('Error: Invalid JSON. Configuration not saved.');
}
});
app.post('/updateconfig', (req, res) => {
let {
payload
} = req.body;
const check_token = req.body.token;
try {
const verify_payload = JSON.parse(req.body.payload);
if ((check_token !== token) || (!check_token)) {
res.end('\nError: Invalid Credentials');
} else {
payload = JSON.stringify(verify_payload, null, 4);
setTimeout(() => {
fs.writeFile(config_file, payload, err => {
if (err) {
console.log('\nError while writing config.' + err);
} else {
copyToAgents(payload, config_file, 'config', '');
reloadConfig();
res.end('Updated Configuration.');
}
});
}, 3000);
}
} catch (error) {
res.end('Error: Invalid JSON. Configuration not saved.');
}
});
if (config.ssl && config.ssl_cert && config.ssl_key) {
console.log('SSL Server API enabled');
const ssl_options = {
cert: fs.readFileSync(config.ssl_cert),
key: fs.readFileSync(config.ssl_key)
};
const server = https.createServer(ssl_options, app);
server.listen(server_port, () => {
console.log('Listening on port %d', server_port);
});
} else {
console.log('Non-SSL Server API enabled');
const server = http.createServer(app);
server.listen(server_port, () => {
console.log('Listening on port %d', server_port);
});
}