From 311a686f3672a00c8def9190c874d6f83a006e35 Mon Sep 17 00:00:00 2001 From: Jannik Schönartz Date: Mon, 6 Aug 2018 01:04:54 +0000 Subject: [server/external-backends] Implemented import Objects from iDoIT renamed external-backends.js in index.js so its return when requireing the folder Added new tables for the external id mapping for clients / groups iDoIT method for importing Objects and adding them in the db with all neccessary constraints --- server/api/backends.js | 138 ++++++++++++++++++++- .../external-backends/backends/another-backend.js | 4 +- .../external-backends/backends/dummy-backend.js | 4 +- .../external-backends/backends/idoit-backend.js | 120 ++++++++++++++---- .../external-backends/backends/template-backend.js | 4 +- server/lib/external-backends/external-backends.js | 93 -------------- server/lib/external-backends/index.js | 104 ++++++++++++++++ server/migrations/20180715193710-create-backend.js | 4 +- .../20180805054700-create-backend_x_client.js | 38 ++++++ .../20180805054700-create-backend_x_group.js | 38 ++++++ server/models/backend.js | 10 +- 11 files changed, 425 insertions(+), 132 deletions(-) delete mode 100644 server/lib/external-backends/external-backends.js create mode 100644 server/lib/external-backends/index.js create mode 100644 server/migrations/20180805054700-create-backend_x_client.js create mode 100644 server/migrations/20180805054700-create-backend_x_group.js diff --git a/server/api/backends.js b/server/api/backends.js index 5e98a9c..2fae6bd 100644 --- a/server/api/backends.js +++ b/server/api/backends.js @@ -1,6 +1,6 @@ /* global __appdir */ const path = require('path') -const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends', 'external-backends.js')) +const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')) var db = require(path.join(__appdir, 'lib', 'sequelize')) // GET requests @@ -128,8 +128,8 @@ module.exports.get = { const id = req.query.id var types = {} db.backend.findOne({ where: { id: id } }).then(backend => { - types.groups = JSON.parse(backend.groups) - types.clients = JSON.parse(backend.clients) + types.groups = JSON.parse(backend.groupTypes) + types.clients = JSON.parse(backend.clientTypes) types.sync = backend.sync res.status(200).send(types) }) @@ -147,7 +147,136 @@ module.exports.get = { const instance = ba.getInstance(backend.type) res.status(200).send(instance.getSyncTypes()) }) + }, + + /* + * ?id: + * id: + */ + importObjects: function (req, res) { + const id = req.query.id + // const id = req.body.id + + // Get the backend where the objects are importet from. + db.backend.findOne({ where: { id: id } }).then(backend => { + if (backend) { + var endRequest = [] + const ba = new ExternalBackends() + const instance = ba.getInstance(backend.type) + const groups = JSON.parse(backend.groupTypes).map(x => x.id) + const clients = JSON.parse(backend.clientTypes).map(x => x.id) + + // Get a list with all objects in the backend. + const objectPromise = new Promise(function (resolve, reject) { + resolve(instance.getObjects(backend.credentials)) + }) + + objectPromise.then(result => { + // Check for the not implemented exception + if (result.status) res.status(501).send(result) + + // Filter those objects in groups / clients + var groupObjects = [] + var clientObjects = [] + result.objects.filter(obj => { + if (groups.find(x => x === obj.type)) groupObjects.push({ id: obj.id, name: obj.title, type: obj.type, typeName: obj.type_title, sysid: obj.sysid }) + else if (clients.find(y => y === obj.type)) clientObjects.push({ id: obj.id, name: obj.title, type: obj.type, typeName: obj.type_title, sysid: obj.sysid }) + }) + + var promises = [] + var promises2 = [] + + // Add all groups in the database. + groupObjects.forEach(group => { + // Insert the group. + promises.push(db.group.create({ name: group.name, description: group.typeName }).then(g => { + // Insert the backend_x_group relation. + promises2.push(backend.addMappedGroups(g, { through: { externalId: group.id, externalType: group.type } })) + })) + }) + + // Add all clients in the databse. + clientObjects.forEach(client => { + // Insert the client. + promises.push(db.client.create({ name: client.name, description: client.typeName }).then(c => { + // Insert the backend_x_client relation. + promises2.push(backend.addMappedClients(c, { through: { externalId: client.id, externalType: client.type } })) + })) + }) + + // Wait till all clients / groups are created and all mapping operations are done. Then add childs. + Promise.all(promises).then(() => { + Promise.all(promises2).then(() => { + // Get the backend with all the mapped groups. ! Only groups can have childs ! + db.backend.findOne({ where: { id: backend.id }, include: ['mappedGroups'] }).then(b => { + var objectData = [] + // Put all groups in the array to make a one session call which returns all needed informations. + b.mappedGroups.forEach(mGroup => { + const mG = mGroup.backend_x_group + const eid = mG.externalId + const gid = mGroup.id + objectData.push({ eid: eid, gid: gid }) + }) + + // Get all the information needed from the backend instance. (For all object ids in the array) + var promise = new Promise(function (resolve) { + resolve(instance.getDataTree(backend.credentials, objectData)) + }) + + promise.then(data => { + // Check for the not implemented exception + if (data.status) res.status(501).send(data) + + data.forEach(obj => { + var groupChildsToAdd = [] + var clientChildsToAdd = [] + var prom = [] + + // Put all clientChilds in the clientList and all groupChilds in the groupList. + obj.childs.forEach(child => { + if (groups.find(x => x === child.type)) { + // Get the group id out of the externalId. + prom.push(db.backend.findOne({ where: { id: backend.id, '$mappedGroups.backend_x_group.externalId$': child.id }, include: ['mappedGroups'] }).then(ba => { + // The externalId should only be once in the db. + if (ba.mappedGroups.length === 1) { + groupChildsToAdd.push(ba.mappedGroups[0].backend_x_group.groupId) + } + })) + } else if (clients.find(x => x === child.type)) { + // Get the client id out of the externalId. + prom.push(db.backend.findOne({ where: { id: backend.id, '$mappedClients.backend_x_client.externalId$': child.id }, include: ['mappedClients'] }).then(ba => { + // The externalId should only be once in the db. + if (ba.mappedClients.length === 1) { + clientChildsToAdd.push(ba.mappedClients[0].backend_x_client.clientId) + } + })) + } + }) + + // After all the group and client ids are collected. Add them as subgroup / client + Promise.all(prom).then(() => { + endRequest.push(db.group.findOne({ where: { id: obj.gid } }).then(group => { + if (group) { + group.addSubgroups(groupChildsToAdd) + group.addClients(clientChildsToAdd) + } + })) + }) + }) + }) + }) + }) + }) + + // If all requests are fullfilled. End the request. + Promise.all(endRequest).then(() => { + res.status(200).send({ status: 'SUCCESS' }) + }) + }) + } else res.status(500).send({ status: 'INVALID_BACKEND_ID', error: 'The provided backend id is invalid.' }) + }) } + } // POST requests @@ -173,7 +302,6 @@ module.exports.post = { // Update an existing backend in the db. db.backend.update({ name: backend.name, type: backend.type, credentials: credentialString }, { where: { id: backend.id } }) } - // db.backend.findOne({}) res.status(200).send('success') }, @@ -232,7 +360,7 @@ module.exports.post = { const clients = JSON.stringify(req.body.clients) const sync = req.body.sync db.backend.findOne({ where: { id: id } }).then(backend => { - db.backend.update({ groups: groups, clients: clients, sync: sync }, { where: { id: id } }).then(() => { + db.backend.update({ groupTypes: groups, clientTypes: clients, sync: sync }, { where: { id: id } }).then(() => { res.status(200).send() }) }) diff --git a/server/lib/external-backends/backends/another-backend.js b/server/lib/external-backends/backends/another-backend.js index fd867a2..0d60259 100644 --- a/server/lib/external-backends/backends/another-backend.js +++ b/server/lib/external-backends/backends/another-backend.js @@ -1,4 +1,6 @@ -var ExternalBackends = require('../external-backends.js') +/* global __appdir */ +const path = require('path') +const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')) class AnotherBackend extends ExternalBackends { getCredentials () { diff --git a/server/lib/external-backends/backends/dummy-backend.js b/server/lib/external-backends/backends/dummy-backend.js index f445a9e..c157e0a 100644 --- a/server/lib/external-backends/backends/dummy-backend.js +++ b/server/lib/external-backends/backends/dummy-backend.js @@ -1,4 +1,6 @@ -var ExternalBackends = require('../external-backends.js') +/* global __appdir */ +const path = require('path') +const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')) class DummyBackend extends ExternalBackends { getCredentials () { diff --git a/server/lib/external-backends/backends/idoit-backend.js b/server/lib/external-backends/backends/idoit-backend.js index 3645710..5d94e41 100644 --- a/server/lib/external-backends/backends/idoit-backend.js +++ b/server/lib/external-backends/backends/idoit-backend.js @@ -1,10 +1,16 @@ -var ExternalBackends = require('../external-backends.js') +/* global __appdir */ +const path = require('path') +const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')) var axios = require('axios') class IdoitBackend extends ExternalBackends { - // Needed functions + // ############################################################################ + // ######################## needed functions ################################# + + /* + * Returns the credential structure / fields, defined in the backends. + */ getCredentials () { - // I do it only needs the API-key. return [ { type: 'text', id: 1, name: 'API url', icon: 'link' }, { type: 'password', id: 2, name: 'API token', icon: 'vpn_key', show: false }, @@ -21,6 +27,12 @@ class IdoitBackend extends ExternalBackends { ] } + /* + * Checks the connection of a given backend. + * idoIT: Checks if we can get a valid session id. If so the connection must be successfull. + * + * return: { success: , status: '', error: '' } + */ async checkConnection (credentials) { var c = this.mapCredentials(credentials) return this.getSession(c) @@ -59,25 +71,6 @@ class IdoitBackend extends ExternalBackends { return ['None', 'Two-Way', 'Upload Only', 'Upload Then Delete', 'Upload Mirror', 'Download Only', 'Download Then Delete', 'Download Mirror'] } - // Optional functions e.g. helperfunctions or testing stuff. - - // Helper function, to map the array of credential objects into a single js object. - mapCredentials (credentials) { - const c = JSON.parse(credentials) - const login = c.find(x => x.id === 3) - var mapped = { - url: c.find(x => x.id === 1).value, - apikey: c.find(x => x.id === 2).value, - login: login.value - } - - if (mapped.login) { - mapped.username = login.elements.find(x => x.id === 4).value - mapped.password = login.elements.find(x => x.id === 5).value - } - return mapped - } - async getObjects (credentials) { var c = this.mapCredentials(credentials) var login = await this.getSession(c) @@ -123,10 +116,86 @@ class IdoitBackend extends ExternalBackends { result.childs = await this.axiosRequest(c.url, 'cmdb.location_tree', params, headers) result.object = result.object.data.result result.childs = result.childs.data.result - return result } + // Function to use the same session for multiple requests + getDataTree (credentials, array) { + var c = this.mapCredentials(credentials) + + // LOGIN + // Open and get a session + const body = { + 'version': '2.0', + 'method': 'idoit.login', + 'params': { + 'apikey': c.apikey, + 'language': 'en' + }, + 'id': 1 + } + + // Headers + var headers = {} + // Optional credentials + if (c.login) { + headers['X-RPC-Auth-Username'] = c.username + headers['X-RPC-Auth-Password'] = c.password + } + + var config = { + timeout: 90000, + headers: headers + } + // Make a login request and see if we are authenticated. + return axios.post(c.url, body, config).then(log => { + var sid = log.data.result['session-id'] + // Headers + config.headers = { 'X-RPC-Auth-Session': sid, 'Content-Type': 'application/json' } + body.method = 'cmdb.location_tree' + var promises = [] + + array.forEach(element => { + const bod = { + 'version': '2.0', + 'method': 'cmdb.location_tree', + 'params': { + 'id': element.eid, + 'apikey': c.apikey, + 'language': 'en' + }, + 'id': 1 + } + promises.push(axios.post(c.url, bod, config).then(result => { + return { gid: element.gid, childs: result.data.result } + })) + }) + return Promise.all(promises).then(ret => { + return ret + }) + }) + } + + // ############################################################################ + // ####################### helper/optional functions ######################### + + // Helper function, to map the array of credential objects into a single js object. + mapCredentials (credentials) { + const c = JSON.parse(credentials) + const login = c.find(x => x.id === 3) + var mapped = { + url: c.find(x => x.id === 1).value, + apikey: c.find(x => x.id === 2).value, + login: login.value + } + + if (mapped.login) { + mapped.username = login.elements.find(x => x.id === 4).value + mapped.password = login.elements.find(x => x.id === 5).value + } + return mapped + } + // Username and password are optional. async getSession (credentials) { // Headers @@ -157,11 +226,10 @@ class IdoitBackend extends ExternalBackends { } var config = { - timeout: 30000, + timeout: 90000, headers: headers } config.headers['Content-Type'] = 'application/json' - var response = await axios.post(url, body, config) .then(response => { return response @@ -174,11 +242,11 @@ class IdoitBackend extends ExternalBackends { if (response.status !== 200) { return { success: false, error: response.status, msg: response.statusText } } - // iDoIT error handling. if (response.data.result) { return { success: true, data: response.data } } else if (response.data.error) { + console.log(response.data.error) return { success: false, error: response.data.error.message, msg: response.data.error.data.error } } else { return { success: false, msg: 'UNNOWN ERROR' } diff --git a/server/lib/external-backends/backends/template-backend.js b/server/lib/external-backends/backends/template-backend.js index 4e5098d..8bee5e1 100644 --- a/server/lib/external-backends/backends/template-backend.js +++ b/server/lib/external-backends/backends/template-backend.js @@ -1,4 +1,6 @@ -var ExternalBackends = require('../external-backends.js') +/* global __appdir */ +const path = require('path') +const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')) class TemplateBackend extends ExternalBackends { getCredentials () { diff --git a/server/lib/external-backends/external-backends.js b/server/lib/external-backends/external-backends.js deleted file mode 100644 index e01a15c..0000000 --- a/server/lib/external-backends/external-backends.js +++ /dev/null @@ -1,93 +0,0 @@ -/* global __appdir */ -const fs = require('fs') -const path = require('path') - -class ExternalBackends { - /* - * Returns a list of all backends which have a .js file. - */ - getBackends () { - var files = fs.readdirSync(path.join(__appdir, 'lib', 'external-backends', 'backends')) - return files - } - - /* - * Returns the credential structure / fields, defined in the backends. - * - * If type === switch there can be child elements in the elements attribute. - * Those elements are only shown, if the switch is set to true. - * [{ type: 'text', id: 1, name: '', icon: '' }, - * { type: 'password', id: 2, name: '', icon: '' }, - * { type: 'switch', id: 3, name: '', icon: '' , elements: [ { type: ... } ] }, - * { type: 'select', id: 4, name: '', icon: '' }, ...] - */ - getCredentials () { - console.log('If this method gets called the backend class has NOT IMPLEMENTED the getCredentials method!') - return null - } - - /* - * type: - * - * Returns the instance of a given backend type. - */ - getInstance (type) { - const bList = this.getBackends() - const bType = type + '-backend.js' - - // Check if it's a valid backend type. - if (bList.indexOf(bType) === -1) { - console.log(bType + ' is not a valid backend type.') - return null - } - - const backend = new (require(path.join(__appdir, 'lib', 'external-backends', 'backends', bType)))() - return backend - } - - /* Returns an empty array [] if the backends doesn't have the function implemented. - * - * return: ['', ...] - */ - getSyncTypes () { - return [] - } - - /* - * credendtials: - * - * Returns a list of all objects in the backend. - */ - async getObjects (credendtials) { - return { status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a getObjects method' } - } - - /* - * credendtials: - * oid: - * - * Call the API of the backend and returns the information to the object including a list of childs. - */ - async getObject (credentials, oid) { - return { status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a getObject method' } - } - - /* - * Checks the connection of a given backend. - * - * return: { success: , status: '', error: '' } - */ - async checkConnection (backend) { - return { success: false, status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a checkConnection method' } - } - - /* Returns an empty array [] if the backends doesn't have such a function. - * - * return: [{id: , title: }, ...] - */ - async getObjectTypes (credentials) { - return [] - } -} - -module.exports = ExternalBackends diff --git a/server/lib/external-backends/index.js b/server/lib/external-backends/index.js new file mode 100644 index 0000000..c882522 --- /dev/null +++ b/server/lib/external-backends/index.js @@ -0,0 +1,104 @@ +/* global __appdir */ +const fs = require('fs') +const path = require('path') + +class ExternalBackends { + /* + * Returns a list of all backends which have a .js file. + */ + getBackends () { + var files = fs.readdirSync(path.join(__appdir, 'lib', 'external-backends', 'backends')) + return files + } + + /* + * Returns the credential structure / fields, defined in the backends. + * + * If type === switch there can be child elements in the elements attribute. + * Those elements are only shown, if the switch is set to true. + * [{ type: 'text', id: 1, name: '<NAME>', icon: '<ICON_NAME>' }, + * { type: 'password', id: 2, name: '<NAME>', icon: '<ICON_NAME>' }, + * { type: 'switch', id: 3, name: '<NAME>', icon: '<ICON_NAME>' , elements: [ { type: ... } ] }, + * { type: 'select', id: 4, name: '<NAME>', icon: '<ICON_NAME>' }, ...] + */ + getCredentials () { + console.log('If this method gets called the backend class has NOT IMPLEMENTED the getCredentials method!') + return null + } + + /* + * type: <BACKEND_TYPE> + * + * Returns the instance of a given backend type. + */ + getInstance (type) { + const bList = this.getBackends() + const bType = type + '-backend.js' + + // Check if it's a valid backend type. + if (bList.indexOf(bType) === -1) { + console.log(bType + ' is not a valid backend type.') + return null + } + + const backend = new (require(path.join(__appdir, 'lib', 'external-backends', 'backends', bType)))() + return backend + } + + /* Returns an empty array [] if the backends doesn't have the function implemented. + * + * return: ['<SYNC_TYPE>', ...] + */ + getSyncTypes () { + return [] + } + + /* + * credendtials: <BACKEND_CREDENTIALS> + * + * Returns a list of all objects in the backend. + */ + async getObjects (credendtials) { + return { status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a getObjects method' } + } + + /* + * credendtials: <BACKEND_CREDENTIALS> + * oid: <OBJECT_ID> + * + * Call the API of the backend and returns the information to the object including a list of childs. + */ + async getObject (credentials, oid) { + return { status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a getObject method' } + } + + /* + * credentials: <BACKEND_CREDENTIALS> + * objects: [{ eid: <EXTERNAL_ID>, gid: <GROUP_ID> }, ...] + * EXTERNAL_ID is the id of the objects in the external backend requestet. + * + * return: [{ gid: <GROUP_ID>, childs: [{ id: <EXTERNAL_ID>, }, ...]}, ...] + */ + getDataTree (credendtials, objects) { + return { status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a getDataTree method' } + } + + /* + * Checks the connection of a given backend. + * + * return: { success: <boolean>, status: '<STATUS_CODE_IF_ERROR>', error: '<ERROR_MESSAGE>' } + */ + async checkConnection (backend) { + return { success: false, status: 'NOT_IMPLEMENTED_EXCEPTION', error: 'The provided backend does not have a checkConnection method' } + } + + /* Returns an empty array [] if the backends doesn't have such a function. + * + * return: [{id: <id>, title: <title>}, ...] + */ + async getObjectTypes (credentials) { + return [] + } +} + +module.exports = ExternalBackends diff --git a/server/migrations/20180715193710-create-backend.js b/server/migrations/20180715193710-create-backend.js index 98f5476..18b324b 100644 --- a/server/migrations/20180715193710-create-backend.js +++ b/server/migrations/20180715193710-create-backend.js @@ -19,12 +19,12 @@ module.exports = { type: Sequelize.STRING(2048), defaultValue: '[]' }, - groups: { + groupTypes: { allowNull: false, type: Sequelize.STRING(4096), defaultValue: '[]' }, - clients: { + clientTypes: { allowNull: false, type: Sequelize.STRING(4096), defaultValue: '[]' diff --git a/server/migrations/20180805054700-create-backend_x_client.js b/server/migrations/20180805054700-create-backend_x_client.js new file mode 100644 index 0000000..ce2c3fe --- /dev/null +++ b/server/migrations/20180805054700-create-backend_x_client.js @@ -0,0 +1,38 @@ +'use strict' +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.createTable('backend_x_client', { + backendId: { + primaryKey: true, + allowNull: false, + type: Sequelize.INTEGER, + onDelete: 'cascade', + references: { + model: 'backends', + key: 'id' + } + }, + clientId: { + allowNull: true, + type: Sequelize.INTEGER, + onDelete: 'SET NULL', + references: { + model: 'clients', + key: 'id' + } + }, + externalId: { + primaryKey: true, + allowNull: true, + type: Sequelize.INTEGER + }, + externalType: { + allowNull: true, + type: Sequelize.STRING + } + }) + }, + down: (queryInterface, Sequelize) => { + return queryInterface.dropTable('backend_x_client') + } +} diff --git a/server/migrations/20180805054700-create-backend_x_group.js b/server/migrations/20180805054700-create-backend_x_group.js new file mode 100644 index 0000000..d187e24 --- /dev/null +++ b/server/migrations/20180805054700-create-backend_x_group.js @@ -0,0 +1,38 @@ +'use strict' +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.createTable('backend_x_group', { + backendId: { + primaryKey: true, + allowNull: false, + type: Sequelize.INTEGER, + onDelete: 'cascade', + references: { + model: 'backends', + key: 'id' + } + }, + groupId: { + allowNull: true, + type: Sequelize.INTEGER, + onDelete: 'SET NULL', + references: { + model: 'groups', + key: 'id' + } + }, + externalId: { + primaryKey: true, + allowNull: true, + type: Sequelize.INTEGER + }, + externalType: { + allowNull: true, + type: Sequelize.STRING + } + }) + }, + down: (queryInterface, Sequelize) => { + return queryInterface.dropTable('backend_x_group') + } +} diff --git a/server/models/backend.js b/server/models/backend.js index 62b936a..f08e6c0 100644 --- a/server/models/backend.js +++ b/server/models/backend.js @@ -14,12 +14,12 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.STRING(2048), defaultValue: '[]' }, - groups: { + groupTypes: { allowNull: false, type: DataTypes.STRING(4096), defaultValue: '[]' }, - clients: { + clientTypes: { allowNull: false, type: DataTypes.STRING(4096), defaultValue: '[]' @@ -31,7 +31,11 @@ module.exports = (sequelize, DataTypes) => { timestamps: false }) backend.associate = function (models) { - // associations can be defined here + var BackendXGroup = sequelize.define('backend_x_group', { externalId: DataTypes.INTEGER, externalType: DataTypes.STRING }, { timestamps: false, freezeTableName: true }) + var BackendXClient = sequelize.define('backend_x_client', { externalId: DataTypes.INTEGER, externalType: DataTypes.STRING }, { timestamps: false, freezeTableName: true }) + + backend.belongsToMany(models.group, { as: 'mappedGroups', through: BackendXGroup, foreignKey: 'backendId', otherKey: 'groupId' }) + backend.belongsToMany(models.client, { as: 'mappedClients', through: BackendXClient, foreignKey: 'backendId', otherKey: 'clientId' }) } return backend } -- cgit v1.2.3-55-g7522