Created
February 13, 2020 22:49
-
-
Save epugh/2e0a591e2d06b5c472bdfa704a27b142 to your computer and use it in GitHub Desktop.
Quepid NDCG@10 developed by Bertrand
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Wrap the Quepid objects and API in a namespace. | |
// Simple pass-through required by our NDCG scorer. | |
let quepidApi = {}; | |
(function(context) { | |
context.getDocs = function() { | |
return docs; | |
} | |
context.getBestDocs = function() { | |
return bestDocs; | |
} | |
// Document's rating accessor | |
context.docRating = function(position) { | |
// console.log('Position ' + position + ' rating: ' + docRating(position)); | |
return docRating(position); | |
} | |
// Has a ranking position been rated? | |
context.hasDocRating = function(position) { | |
// console.log('Position ' + position + ' rating? ' + hasDocRating(position)); | |
return hasDocRating(position); | |
} | |
})(quepidApi); | |
// Scorer's name space to avoid name collision with the Quepid-provided objects. | |
var NDCG_scorer = {}; | |
(function(context) { | |
let _quepidApi = null; | |
// Logging function. | |
function log(obj) { | |
// Log messages on the developer's console. | |
var debug = true | |
if (debug === true) console.log(obj) | |
} | |
// -------------------------------------------------------------------- | |
// Retrieve a document's rating. | |
// hasDocRating(): See mock above. | |
// docRating(): See mock above. | |
// -------------------------------------------------------------------- | |
function ratingOrDefault(posn, defVal) { | |
if (!_quepidApi.hasDocRating(posn)) { | |
return defVal; | |
} | |
return _quepidApi.docRating(posn); | |
} | |
// -------------------------------------------------------------------- | |
// Log base 2. | |
// -------------------------------------------------------------------- | |
function log2(num) { | |
return Math.log(num) / Math.log(2); | |
} | |
// -------------------------------------------------------------------- | |
// Round with precision. Used for logging numbers. | |
// -------------------------------------------------------------------- | |
function myRound(number, precision = 0) { | |
const factor = Math.pow(10, precision); | |
const tempNumber = number * factor; | |
const roundedTempNumber = Math.round(tempNumber); | |
return roundedTempNumber / factor; | |
} | |
// -------------------------------------------------------------------- | |
// Build the axtual list of ratings. | |
// Arguments: | |
// p: Max rank position (See 'p' in the Wikipedia article) | |
// -------------------------------------------------------------------- | |
function actualRatings(rankPosition) { | |
const ratings = []; | |
for (let i = 0; i < rankPosition; i++) { | |
const rating = ratingOrDefault(i, 0.0) | |
// log("Doc " + i + ": " + rating) | |
ratings.push(rating); | |
} | |
log("Actual Ratings: " + ratings); | |
return ratings; | |
} | |
// -------------------------------------------------------------------- | |
// Build the ideal list of ratings. | |
// Arguments: | |
// p: Max rank position (See 'p' in the Wikipedia article) | |
// -------------------------------------------------------------------- | |
function idealRatings(rankPosition) { | |
const idealRatings = []; | |
const bestDocsCount = _quepidApi.getBestDocs().length; | |
for (let i = 0; i < bestDocsCount && i < rankPosition; i++) { | |
idealRatings.push(_quepidApi.getBestDocs()[i].rating); | |
} | |
// Padd with 0 (Not rated) if necessary. | |
for (let i = idealRatings.length; i < rankPosition; i++) { | |
// log("Ideal ratings list: Padding with 0.0 at position (0-based) " + i); | |
idealRatings.push(0.0); | |
} | |
log("Ideal ratings: " + idealRatings); | |
return idealRatings; | |
} | |
// -------------------------------------------------------------------- | |
// DCG calculator | |
// Arguments: | |
// docList: List of rated documents. | |
// p: Maximum rank position. | |
// -------------------------------------------------------------------- | |
function dcg(ratings, p) { | |
let dcgScore = 0 | |
for (var i = 1; i <= ratings.length && i <= p; i++) { | |
const logPosition = log2(i + 1); | |
const dcgAdder = ratings[i - 1] / logPosition | |
dcgScore += dcgAdder | |
log('i=' + i + " ratings[i-1]=" + ratings[i - 1] + "/log(i+1)=" + logPosition + " => DCG incr. of " + myRound(dcgAdder, 3) + " (DCG=" + myRound(dcgScore, 3) + ")") | |
} | |
log('DCG([' + ratings + ']) = ' + dcgScore) | |
return dcgScore | |
} | |
// Public API: ndcg | |
// Arguments: | |
// 1. Ranking position | |
// For example, for an NDCG@10, use p = 10 | |
// | |
// ATTENTION: Due to current limitation in Quepid ("bestDocs" is | |
// 10-deep at most), p must be <= max(Number of results shown, 10) | |
// For example, for p = 10, set the "Number of Results to Show" | |
// in the Settings to 10. | |
// | |
// 2. Rating scale max value: MUST MATCH THE MAX RATING SPECIFIED IN THE | |
// "Scale for query ratings" scorer configuration. | |
context.ndcg = function(quepidApi, rankPosition, ratingScaleMax) { | |
log(" RUNNING NDCG!!! "); | |
log(" *************** "); | |
_quepidApi = quepidApi; | |
log('Docs count : ' + _quepidApi.getDocs().length) | |
log('bestDocs count: ' + _quepidApi.getBestDocs().length) | |
// Log the objects for inspection in the developer's console | |
log(_quepidApi.getDocs()) | |
log(_quepidApi.getBestDocs()) | |
log('Rank position (p)=' + String(rankPosition)) | |
log('Using scale 1-' + String(ratingScaleMax)) | |
var myDcg = dcg(actualRatings(rankPosition), rankPosition); | |
var iDcg = dcg(idealRatings(rankPosition), rankPosition); | |
// NDCG is ~ dcg(currDocs()) / dcg(bestDocs()) | |
var nDcg = myDcg / iDcg; | |
log(" DCG: " + myDcg); | |
log("iDCG: " + iDcg); | |
log("NDCG: " + nDcg); | |
return nDcg; | |
} | |
})(NDCG_scorer); | |
setScore(NDCG_scorer.ndcg(quepidApi, 10, 10) * 100); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment