Skip to content

Instantly share code, notes, and snippets.

@alexbfree
Created June 29, 2016 09:23
Show Gist options
  • Save alexbfree/f9951b4bf4d4c3c59f2ea0c6068a25c2 to your computer and use it in GitHub Desktop.
Save alexbfree/f9951b4bf4d4c3c59f2ea0c6068a25c2 to your computer and use it in GitHub Desktop.
connect to Panoptes JS API and remove subjects
#!/usr/bin/env node
require('es6-promise').polyfill();
// staging
//var PROJECTID = "1611";
//var WORKFLOWID = "2315";
//var SUBJECTSETID = "3700";
//var SUBJECT_IDS_TO_DELETE = ["45046"];
// production
var PROJECTID = "988";
var WORKFLOWID = "512";
var SUBJECTSETID = "4282";
var SPECIES_TO_DELETE = ["Hartebeest", "Wildebeest", "Buffalo", "Guinea Fowls", "Warthogs"];
var SUBJECT_IDS_TO_DELETE = [];
var FastCSV = require('fast-csv');
var fs = require('fs');
var glob = require('glob');
var argv = require('yargs').argv;
var request = require('request');
var auth = require('panoptes-client/lib/auth');
var api = require('panoptes-client/lib/api-client');
var subject_set_name = null;
var workflow_name = null;
var csv_name = argv.csv;
if (typeof(csv_name)=="undefined") {
throw new Error("Please launch with \"node remove-subjects.js -u username -p password --csv <your_csv_filename>\"");
}
var input = fs.createReadStream( csv_name );
var count = 0;
function processLine(line) {
workflow_id = (eval(line.workflow_ids)[0]).toString();
metadata = JSON.parse(line.metadata);
species = metadata["Crowd Determination"]
if (PROJECTID==line.project_id && WORKFLOWID==workflow_id && SUBJECTSETID==line.subject_set_id && SPECIES_TO_DELETE.indexOf(species) >= 0) {
SUBJECT_IDS_TO_DELETE.push(line.subject_id);
}
count = count + 1;
}
// this should work and tell us staging or production:
// console.log ("Connected to ",api.host);
// (can change by setting NODE_ENV=staging or NODE_ENV=production)
function signIn() {
login = {
login: argv.u,
password: argv.p,
remember_me: true
}
return auth.signIn(login);
}
function testGetWorkflow() {
console.log(`Doing a test GET of workflow ${WORKFLOWID}...`);
api.type('workflows').get(WORKFLOWID)
.then( function (workflow) {
workflow_name = workflow.display_name
console.log(` Got workflow ${workflow.id}: '${workflow_name}'`);
})
.catch(function (error) {
console.log(` Error getting workflow: ${error}`);
});
}
function testGetSubjectSet() {
console.log(`Doing a test GET of subject set ${SUBJECTSETID}...`);
api.type('subject_sets').get(SUBJECTSETID)
.then( function (subject_set) {
subject_set_name = subject_set.display_name
console.log(` Got subject set ${subject_set.id}: '${subject_set_name}'`);
})
.catch(function (error) {
console.log(` Error getting subject set ${error}`);
});
}
function testDeleteSubjects() {
console.log(`Doing a test DELETE from subject set ${SUBJECTSETID} of subjects ${SUBJECT_IDS_TO_DELETE}...`);
api.type('subject_sets').get(SUBJECTSETID)
.then( function (subject_set) {
subject_set_name = subject_set.display_name;
console.log(` Got subject set ${subject_set.id}: '${subject_set_name}', which currently has ${subject_set.set_member_subjects_count} subjects. Deleting...`);
subject_set.removeLink('subjects', SUBJECT_IDS_TO_DELETE)
.then( function(new_subject_sets) {
new_subject_set = new_subject_sets[0]
console.log(` Deleted subjects successfully. Set now has ${new_subject_set.set_member_subjects_count} subjects remaining.`);
})
.catch( function(e) {
console.log(e);
});
})
.catch(function (error) {
console.log(`Error getting subject set ${SUBJECTSETID} ready to delete from it`, error);
});
}
function testGetSubjectMessageSet() {
console.log(`Doing a test get of subject message set for subject set ${SUBJECTSETID} and workflow '${WORKFLOWID}...`);
api.type('set_member_subjects').get({subject_set_id: SUBJECTSETID, workflow: WORKFLOWID})
.then( function (setMemberSubjects) {
console.log(`Found subject message set for subject set '${subject_set_name}' and workflow '${workflow_name}' with ${setMemberSubjects.length} linked subjects.`);
console.log("Getting subjects...");
setMemberSubjects.forEach( function (setMemberSubject) {
sms_id = setMemberSubject.id;
subject_id = setMemberSubject.links.subject;
// note these are equal
doIt = true;
subjectsToDelete=[];
api.type('subjects').get(subject_id)
.then( function (subject) {
console.log(` ID: ${subject.id}`);
console.log(" Media:");
subject.locations.forEach( function (location) {
for (var type in location) {
if (location.hasOwnProperty(type)) {
console.log(` ${location[type]} (${type})`);
}
}
});
console.log(" Metadata:");
console.log(` ${JSON.stringify(subject.metadata)}`);
})
.catch(function (error) {
console.log(`Error getting subject ${subject_id}`, error);
});
api.type('subject_sets').
testDeleteSubject(setMemberSubjects,subjectsToDelete);
});
})
.catch(function (error) {
console.log(`Error getting subject message set: {$error}`);
});
}
function deleteSubjects(subject_IDs_to_delete) {
console.log(subject_IDs_to_delete);
api.type('subject_sets').get(SUBJECTSETID)
.then( function (subject_set) {
subject_set_name = subject_set.display_name;
console.log(` Got subject set ${subject_set.id}: '${subject_set_name}', which currently has ${subject_set.set_member_subjects_count} subjects. Deleting ${subject_IDs_to_delete.length} subjects...`);
subject_set.removeLink('subjects', subject_IDs_to_delete)
.then( function(new_subject_sets) {
new_subject_set = new_subject_sets[0];
console.log(` Successfully submitted request to delete subjects. Set currently has ${new_subject_set.set_member_subjects_count} subjects remaining (but check back soon and they all will have been deleted.`);
})
.catch( function(e) {
console.log(e);
});
})
.catch(function (error) {
console.log(`Error getting subject set ${SUBJECTSETID} ready to delete from it`, error);
});
}
function processedCSV() {
console.log(`Processed CSV. Processed ${count} lines and found ${SUBJECT_IDS_TO_DELETE.length} subject IDs to be deleted.`);
var chunkOfBigSubjectIDsList = [];
var chunkSize = 100;
for (i=0, j=SUBJECT_IDS_TO_DELETE.length; i<j; i+=chunkSize) {
chunkOfBigSubjectIDsList = SUBJECT_IDS_TO_DELETE.slice(i, i + chunkSize);
deleteSubjects(chunkOfBigSubjectIDsList);
}
console.log ("\nDone.\n");
}
signIn()
.then( function (user) {
api.update({'params.admin' : user.admin})
Promise.all([
api.type('workflows').get(WORKFLOWID),
api.type('projects').get(PROJECTID),
api.type('subject_sets').get(SUBJECTSETID)
])
.then( function (values) {
console.log(`WORKFLOW ${values[0].display_name}`);
console.log(`PROJECT ${values[1].display_name}`);
console.log(`SUBJECT SET ${values[2].display_name}`);
subject_set_name = values[2].display_name;
//testGetSubjectSet();
//testGetWorkflow();
//testGetSubjectMessageSet();
//testDeleteSubjects();
var stream = fs.createReadStream(csv_name)
FastCSV
.fromStream(stream, {headers : true})
.on("data", processLine )
.on("end", processedCSV );
})
.catch(function (error) {
console.log(`Error getting workflow and project: ${error}`);
});
})
.catch(function(error){
console.log('Done', error);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment