Last active
June 15, 2023 16:27
-
-
Save ChecksumFailed/205270ed10ad170b5c432697011a884c to your computer and use it in GitHub Desktop.
DataTransformUtilsV2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* NOTE; This is a work in progress and has not been tested in its current form. | |
* Set of methods to simplify loading data into ServiceNow. Use with Inbound Email, rest, etc... | |
* A temporary DS/importset will be created for the data load. It will be removed if the cleanup method is called. | |
* Example usage: DataTransformUtils().getDataSourceByName('Test DataSource').copyDataSource().copyAtt(sourceRecord).loadData().getMapByName('Test Transform Map').importData().cleanUpImport().log(); | |
* this will: | |
* 1. Retrieve Existing DS by name | |
* 2. Create a clone of this datasource | |
* 3. Copy attachments from a source record to copied datasource | |
* 4. load data from the datasource into the import set table | |
* 5 Retrieve transform map to use | |
* 6. Transform loaded data | |
* 7. Cleanup temporary ds | |
* 8. Log the steps to the system log | |
*/ | |
var DataTransformUtils = function () { | |
var logger = new GSLog("cf.dataTransFormUtils.log", "DataTransformUtils"); | |
/** | |
* create a new datasource from a mapping of field:value | |
* @param {Array} dataSourceMap | |
* @returns {GlideRecord} Datasource | |
*/ | |
function createDataSource(dataSourceMap) { | |
_argumentValidation( | |
dataSourceMap, | |
_objectCheck, | |
"dataSourceMap variable must be an object containing field name and value pairs" | |
); | |
var grDatasource = new GlideRecord("sys_data_source"); | |
for (var key in dataSourceMap) { | |
grDatasource.setValue(key, dataSourceMap[key]); | |
} | |
var ts = _getTimeStamp(); | |
grDatasource.name = grDS.name + ts; | |
grDatasource.insert(); | |
logger.logDebug("Created DataSource - " + datasource.name); | |
return grDatasource; | |
} | |
/** | |
* generate timestamp from current date | |
* @returns {string} | |
*/ | |
function _getTimeStamp() { | |
var dateTimeObj = new GlideDateTime(); | |
return dateTimeObj.getByFormat("yyyyMMddHHmmss"); | |
} | |
/** | |
* Get DataSource record by sys_id | |
* @param {String} sysid | |
* @returns {GlideRecord} | |
*/ | |
function getDataSourceBySysID(sysid) { | |
_argumentValidation( | |
sysid, | |
_notNil, | |
"Valid sysid required" | |
); | |
grDatasource = _getRecord("sys_data_source", sysid); | |
return grDatasource; | |
} | |
/** | |
* get DataSource record by name | |
* @param {String} name | |
* @returns {GlideRecord} | |
*/ | |
function getDataSourceByName(name) { | |
_argumentValidation( | |
name, | |
_notNil, | |
"Name is required when calling getDataSourceByName" | |
); | |
var qry = [ | |
{ | |
field: "name", | |
value: name, | |
}, | |
{ | |
field: "active", | |
value: true, | |
}, | |
]; | |
var grDatasource = _qryRecord("sys_data_source", qry); | |
return grDatasource; | |
} | |
/** | |
* Get Transform Map by SysID | |
* @param {String} sysid | |
* @returns this | |
*/ | |
function getMapBySysID(sysid) { | |
_argumentValidation( | |
sysid, | |
_notNil, | |
"Valid sysid required" | |
); | |
var map = _getRecord("sys_transform_map", sysid); | |
_updateStatus("Retrieved Transform Map - " + map.name); | |
return this; | |
} | |
/** | |
* Get Transform Map by Name | |
* @param {String} name | |
* @returns {GlideRecord} | |
*/ | |
function getMapByName(name) { | |
_argumentValidation( | |
name, | |
_notNil, | |
"Transform Map name must not be empty" | |
); | |
var qry = [ | |
{ | |
field: "name", | |
value: name, | |
}, | |
{ | |
field: "active", | |
value: true, | |
}, | |
]; | |
var grTransformMap = _qryRecord("sys_transform_map", qry); | |
logger.logDebug("Retrieved Transform Map - " + map.name); | |
return grTransformMap; | |
} | |
/** | |
* Get GlideRecord by Sys_ID | |
* @param {String} tbl | |
* @param {String} sysid | |
* @returns {GlideRecord} | |
*/ | |
function _getRecord(tbl, sysid) { | |
_argumentValidation( | |
[tbl, sysid], | |
_notNil, | |
"Table and sysid variables must not be empty and valid" | |
); | |
var grRec = new GlideRecord(tbl); | |
grRec.get(sysid); | |
return grRec; | |
} | |
/** | |
* Get GlideRecord by qry | |
* @param {String} tbl | |
* @param {Array} queries | |
* @returns {GlideRecord} | |
*/ | |
function _qryRecord(tbl, queries) { | |
var qryObj; | |
var exampleObj = [ | |
{ | |
field: "name", | |
value: name, | |
}, | |
{ | |
field: "active", | |
value: true, | |
}, | |
]; | |
_argumentValidation( | |
[tbl, sysid], | |
_notNil, | |
"Table and queries variables must not be empty\nExample Query Obj:\n" + | |
JSON.stringify(exampleObj) | |
); | |
_argumentValidation( | |
queries, | |
Array.isArray, | |
"Queries variable must be an Array of objects containing field name , operator, and value" | |
); | |
var grRec = new GlideRecord(tbl); | |
for (var i = 0; i++; i < queries.length) { | |
qryObj = queries[i]; | |
grRec.addQuery( | |
qryObj["field"], | |
qryObj["operator"] || "=", | |
qryObj["value"] | |
); | |
} | |
grRec.setLimit(1); | |
grRec.query(); | |
return grRec.next() ? grRec : null; | |
} | |
/** | |
* Copy attachments from source record to datasource | |
* @param {GlideRecord} grAttSourceRecord | |
* @param {GlideRecord} grDstDatasource | |
* @returns {Array} | |
*/ | |
function copyAtt(grAttSourceRecord, grDatasource) { | |
_argumentValidation( | |
[grSrcDataSource, grDstDatasource], | |
_notNil, | |
"Source record and Datasource record must be valid" | |
); | |
var attachments = new GlideSysAttachment().copy( | |
grAttSourceRecord.getRecordClassName(), | |
grAttSourceRecord.getValue("sys_id"), | |
grDstDatasource.getRecordClassName(), | |
grDstDatasource.getValue("sys_id") | |
); | |
logger.logDebug("Copied Attachments from " + grAttSourceRecord.name + " to " + grDstDatasource.name); | |
return attachments.join(","); | |
} | |
/* Legacy | |
function importData(grDataSource, grTransformMap) { | |
_argumentValidation([grSource,grDatasource],_notNil,"Valide DataSource GlideRecord and TrasformMap GlideRecord required"); | |
_loadData(grDataSource)._transformData(grTransformMap); | |
return this; | |
} | |
*/ | |
/** | |
* Copy an existing datasource to a new DataSource for one time import | |
* @param {GlideRecord} grDataSource | |
* @returns this | |
*/ | |
function copyDataSource(grDataSource) { | |
_argumentValidation( | |
[grDatasource], | |
_notNil, | |
"Datasource argument required" | |
); | |
var dsObj = {}; | |
for (var key in grDataSource) { | |
if (/^sys_/gi.test(key) == false) { | |
dsObj[key] = grDataSource[key]; | |
} | |
} | |
logger.logDebug("Copying Datasource: " + datasource.name); | |
var grNewDataSource = createDataSource(dsObj); | |
return grNewDataSource; | |
} | |
/** | |
* Transform Loaded Data | |
* @param {GlideRecord} grTransformMap | |
* @param {GlideRecord} grImportSet | |
* @returns {GlideRecord} | |
*/ | |
function transformData(grImportSet, grTransformMap) { | |
_argumentValidation( | |
grImportSet, | |
_notNil, | |
"ImportSet argument required" | |
); | |
var importSetTransformer = new GlideImportSetTransformer(); | |
importSetTransformer.transformAllMaps(grImportSet); | |
logger.logDebug("Imported Data using transform map " + transformMap.name); | |
return grImportSet; | |
} | |
/** | |
* Load Data from datasource | |
* @param {GlideRecord} grDatasource DataSource | |
* @param {GlideRecord} grImportSet | |
* @returns | |
*/ | |
function loadData(grDatasource, grImportSet) { | |
_argumentValidation( | |
[grDatasource, grImportSet], | |
_notNil, | |
"Datasource and Importset required" | |
); | |
var loader = new GlideImportSetLoader(); | |
var importSetGr = loader.getImportSetGr(grDatasource); | |
var ranload = loader.loadImportSetTable(grImportSet, grDatasource); | |
if (!ranLoad) { | |
throw "Failed to load import set " + grDatasource.name; | |
} | |
logger.logDebug("Loaded Data using dataorce " + grDatasource.name); | |
return; | |
} | |
/** | |
* Checks if variable is null, undefined, empty | |
* @param varToCheck | |
*/ | |
function _argumentValidation(args, validationFunction, msg) { | |
msg = msg || "All arguments required"; | |
for (var argIdx = 0; argIdx < arguments.length; argIdx++) { | |
if (validationFunction(arguments[argIdx]) === false) { | |
throw msg; | |
} | |
} | |
} | |
/** | |
* Checks if variable is an Object | |
* @param varToCheck | |
* @returns {boolean} | |
*/ | |
function _isObject(varToCheck) { | |
return typeof varToCheck === "object"; | |
} | |
/** | |
* Check if variable is null,undefined, blank | |
* @param {*} varToCheck | |
* @returns {boolean} | |
*/ | |
function _notNil(varToCheck) { | |
return !gs.nil(varToCheck); | |
} | |
function _isValidSysID(sysid) { | |
return GlideStringUtil.isEligibleSysID(sysid); | |
} | |
/** | |
* Cleans up after data load | |
* @returns | |
*/ | |
function cleanImport(grImportSet) { | |
if (_isEmpty(grImportSet) { | |
throw "Post Import cleanup can not run until after import"; | |
} | |
var cleaner = new ImportSetCleaner(grImportSet.getRecordClassName()); | |
cleaner.setDataOnly(true); | |
cleaner.setDeleteMaps(false); | |
cleaner.clean(); | |
logger.logDebug("Cleaned up import"); | |
return; | |
} | |
/* | |
* Remove temporary datasource | |
*returns this | |
*/ | |
function CleanUpTempData(grDatasource) { | |
if (grDatasource) { | |
grDatasource.deleteRecord(); | |
} | |
return; | |
} | |
/************************************* | |
******* Public Methods ********** | |
************************************/ | |
return { | |
getDataSourceBySysID: getDataSourceBySysID, | |
getDataSourceByName: getDataSourceByName, | |
createDataSource: createDataSource, | |
copyDataSource: copyDataSource, | |
copyAttToDataSource: copyAtt, | |
getMapBySysID: getMapBySysID, | |
getMapByName: getMapByName, | |
loadData: loadData, | |
importData: transformData, | |
cleanUp: CleanUpTempData, | |
type: "DataTransformUtils", | |
}; | |
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function pipe() { | |
var fns = Array.prototype.slice.call(arguments); | |
return function(result) { | |
return fns.reduce(function(value, fn) { | |
return fn(value); | |
}, result); | |
}; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment