summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJannik Schönartz2018-11-04 16:56:53 +0100
committerJannik Schönartz2018-11-04 16:56:53 +0100
commit020ef34fcb9022e4528d6dc051e618ce19e2c943 (patch)
treef2478f9d56e0a0e23db3c004c11211529ab86db6
parenteslint fix (diff)
downloadbas-020ef34fcb9022e4528d6dc051e618ce19e2c943.tar.gz
bas-020ef34fcb9022e4528d6dc051e618ce19e2c943.tar.xz
bas-020ef34fcb9022e4528d6dc051e618ce19e2c943.zip
[iDoIT] Rework the import method to use batch requests
The use of batch request reduce the import time from 1h 10+ min to about 5 min.
-rw-r--r--server/api/backends.js34
-rw-r--r--server/lib/external-backends/backends/idoit-backend.js51
2 files changed, 74 insertions, 11 deletions
diff --git a/server/api/backends.js b/server/api/backends.js
index 77506ba..7729588 100644
--- a/server/api/backends.js
+++ b/server/api/backends.js
@@ -4,6 +4,7 @@ const ExternalBackends = require(path.join(__appdir, 'lib', 'external-backends')
var db = require(path.join(__appdir, 'lib', 'sequelize'))
var express = require('express')
var router = express.Router()
+var noAuthRouter = express.Router()
// GET requests.
@@ -119,7 +120,38 @@ router.get('/:id/synctypes', (req, res) => {
})
})
-// POST requests
+// vvvv TEST MEHTOD !!!! <----------------------------------------------------------
+noAuthRouter.get('/:id/test/getObjectTree', (req, res) => {
+ const id = req.params.id
+ db.backend.findOne({ where: { id: id } }).then(backend => {
+ if (backend) {
+ const ba = new ExternalBackends()
+ const instance = ba.getInstance(backend.type)
+
+ // Get the backend with all the mapped groups. ! Only groups can have childs !
+ db.backend.findOne({ where: { id: backend.id }, include: ['mappedGroups'] }).then(b => {
+ var objectData = []
+ // Put all groups in the array to make a one session call which returns all needed informations.
+ b.mappedGroups.forEach(mGroup => {
+ const mG = mGroup.backend_x_group
+ const eid = mG.externalId
+ const gid = mGroup.id
+ objectData.push({ eid: eid, gid: gid })
+ })
+ // Get all the information needed from the backend instance. (For all object ids in the array)
+ var promise = new Promise(function (resolve) {
+ // resolve(instance.getDataTree(backend.credentials, objectData.slice(0, 200)))
+ resolve(instance.getDataTree(backend.credentials, objectData))
+ })
+ promise.then(data => {
+ res.send({ length: data.length, result: data })
+ })
+ })
+ } else res.send({ error: 'error' })
+ })
+})
+module.exports.noAuthRouter = noAuthRouter
+// ^^^^ TEST MEHTOD !!!! <----------------------------------------------------------
/*
* id: <BACKEND_ID>
diff --git a/server/lib/external-backends/backends/idoit-backend.js b/server/lib/external-backends/backends/idoit-backend.js
index a48c740..dc5a5d8 100644
--- a/server/lib/external-backends/backends/idoit-backend.js
+++ b/server/lib/external-backends/backends/idoit-backend.js
@@ -185,7 +185,7 @@ class IdoitBackend extends ExternalBackends {
}
// Function to use the same session for multiple requests
- async getDataTree (credentials, array) {
+ async getDataTree (credentials, objects) {
var c = this.mapCredentials(credentials)
// LOGIN
@@ -224,9 +224,23 @@ class IdoitBackend extends ExternalBackends {
// Go through the objects and get all the childs.
var promises = []
+
+ var counter = 0
+ var index = 0
+ var gids = {}
+
+ // Prepare all the batch request bodies.
var e
- for (e in array) {
- var element = array[e]
+ for (e in objects) {
+ // Pack 400 requests in one batch request to reduce api calls.
+ if (counter >= 400) {
+ counter = 0
+ index++
+ }
+ if (counter === 0) promises[index] = []
+ counter++
+
+ var element = objects[e]
const bod = {
'version': '2.0',
'method': 'cmdb.location_tree',
@@ -235,16 +249,33 @@ class IdoitBackend extends ExternalBackends {
'apikey': c.apikey,
'language': 'en'
},
- 'id': 1
+ 'id': element.eid
}
+ promises[index].push(bod)
+ gids[element.eid] = element.gid
+ }
- var tmp = await axios.post(c.url, bod, config)
- promises.push({ gid: element.gid, childs: tmp.data.result })
-
- // Add a delay, so that idoit can handle it.
- // await new Promise(resolve => { setTimeout(() => { resolve() }, 500) })
+ // Send all the batch request and post proccess the result into one array.
+ var result = []
+ var p
+ var counter2 = 1
+ for (p in promises) {
+ // TODO: Remove
+ // Counter for getting an overview, how far the requests are.
+ console.log(counter2 + '/' + promises.length + ' requests send')
+ counter2++
+
+ // Send the request.
+ var requestResult = await axios.post(c.url, promises[p], config)
+ const args = requestResult.data
+
+ // Post process the data.
+ var a
+ for (a in args) {
+ result.push({ gid: gids[args[a].id], childs: args[a].result })
+ }
}
- return promises
+ return result
}
// ############################################################################