// ==UserScript== // @name MTurk HIT Database Mk.II // @author feihtality // @namespace https://greasyfork.org/en/users/12709 // @version 0.8.023 // @description Keep track of the HITs you've done (and more!) // @include /^https://www\.mturk\.com/mturk/(dash|view|sort|find|prev|search).*/ // @exclude https://www.mturk.com/mturk/findhits?*hit_scraper // @grant none // @downloadURL none // ==/UserScript== /**\ ** ** This is a complete rewrite of the MTurk HIT Database script from the ground up, which ** eliminates obsolete methods, fixes many bugs, and brings this script up-to-date ** with the current modern browser environment. ** \**/ /* * TODO * optimize searching: index -> date filter * rewrite error handling * tagging (?) * refine searching via R/T buttons * import from old csv format * */ const DB_VERSION = 2; const DB_NAME = 'HITDB_TESTING'; const MTURK_BASE = 'https://www.mturk.com/mturk/'; //const TO_BASE = 'http://turkopticon.ucsd.edu/api/multi-attrs.php'; // polyfill for chrome until v45(?) if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator]; // format leading zeros Number.prototype.toPadded = function(length) { 'use strict'; length = length || 2; return ("0000000"+this).substr(-length); }; // decimal rounding Math.decRound = function(v, shift) { 'use strict'; v = Math.round(+(v+"e"+shift)); return +(v+"e"+-shift); }; Date.prototype.toLocalISOString = function() { 'use strict'; var pad = function(num) { return Number(num).toPadded(); }, offset = pad(Math.floor(this.getTimezoneOffset()/60)) + pad(this.getTimezoneOffset()%60), timezone = this.getTimezoneOffset() > 0 ? "-" + offset : "+" + offset; return this.getFullYear() + "-" + pad(this.getMonth()+1) + "-" + pad(this.getDate()) + "T" + pad(this.getHours()) + ":" + pad(this.getMinutes()) + ":" + pad(this.getSeconds()) + timezone; }; /***********************************************************************************************/ var qc = { extraDays: !!localStorage.getItem("hitdb_extraDays") || false, seen: {} }, metrics = {}; if (localStorage.getItem("hitdb_fetchData")) qc.fetchData = JSON.parse(localStorage.getItem("hitdb_fetchData")); else qc.fetchData = {}; var HITStorage = { //{{{ data: {}, versionChange: function hsversionChange() { //{{{ 'use strict'; var db = this.result; db.onerror = HITStorage.error; db.onversionchange = function(e) { console.log("detected version change??",console.dir(e)); db.close(); }; this.onsuccess = function() { db.close(); }; var dbo; console.groupCollapsed("HITStorage.versionChange::onupgradeneeded"); if (!db.objectStoreNames.contains("HIT")) { console.log("creating HIT OS"); dbo = db.createObjectStore("HIT", { keyPath: "hitId" }); dbo.createIndex("date", "date", { unique: false }); dbo.createIndex("requesterName", "requesterName", { unique: false}); dbo.createIndex("title", "title", { unique: false }); dbo.createIndex("reward", "reward", { unique: false }); dbo.createIndex("status", "status", { unique: false }); dbo.createIndex("requesterId", "requesterId", { unique: false }); localStorage.setItem("hitdb_extraDays", true); qc.extraDays = true; } if (!db.objectStoreNames.contains("STATS")) { console.log("creating STATS OS"); dbo = db.createObjectStore("STATS", { keyPath: "date" }); } if (this.transaction.objectStore("STATS").indexNames.length < 5) { // new in v5: schema additions this.transaction.objectStore("STATS").createIndex("approved", "approved", { unique: false }); this.transaction.objectStore("STATS").createIndex("earnings", "earnings", { unique: false }); this.transaction.objectStore("STATS").createIndex("pending", "pending", { unique: false }); this.transaction.objectStore("STATS").createIndex("rejected", "rejected", { unique: false }); this.transaction.objectStore("STATS").createIndex("submitted", "submitted", { unique: false }); } (function _updateNotes(dbt) { // new in v5: schema change if (!db.objectStoreNames.contains("NOTES")) { console.log("creating NOTES OS"); dbo = db.createObjectStore("NOTES", { keyPath: "id", autoIncrement: true }); dbo.createIndex("hitId", "hitId", { unique: false }); dbo.createIndex("requesterId", "requesterId", { unique: false }); dbo.createIndex("tags", "tags", { unique: false, multiEntry: true }); dbo.createIndex("date", "date", { unique: false }); } if (db.objectStoreNames.contains("NOTES") && dbt.objectStore("NOTES").indexNames.length < 3) { _mv(db, dbt, "NOTES", "NOTES", _updateNotes); } })(this.transaction); if (db.objectStoreNames.contains("BLOCKS")) { console.log("migrating BLOCKS to NOTES"); var temp = []; this.transaction.objectStore("BLOCKS").openCursor().onsuccess = function() { var cursor = this.result; if (cursor) { temp.push( { requesterId: cursor.value.requesterId, tags: "Blocked", note: "This requester was blocked under the old HitDB. Blocking has been deprecated and removed "+ "from HIT Databse. All blocks have been converted to a Note." } ); cursor.continue(); } else { console.log("deleting blocks"); db.deleteObjectStore("BLOCKS"); for (var entry of temp) this.transaction.objectStore("NOTES").add(entry); } }; } function _mv(db, transaction, source, dest, fn) { //{{{ var _data = []; transaction.objectStore(source).openCursor().onsuccess = function() { var cursor = this.result; if (cursor) { _data.push(cursor.value); cursor.continue(); } else { db.deleteObjectStore(source); fn(transaction); if (_data.length) for (var i=0;i<_data.length;i++) transaction.objectStore(dest).add(_data[i]); //console.dir(_data); } }; } //}}} console.groupEnd(); }, // }}} versionChange error: function(e) { //{{{ 'use strict'; if (e === "DatabaseCreationError") { var s = document.getElementById("hdbStatusText"); s.style.color = "red"; s.innerHTML = "Something went wrong during database creation!
Please refresh the page and try again"; console.log("Writing failed with",e); return; } if (typeof e === "string") console.log(e); else console.log("Encountered",e.target.error.name,"--",e.target.error.message,e); }, //}}} onerror parseDOM: function(doc) {//{{{ 'use strict'; var statusLabel = document.querySelector("#hdbStatusText"); statusLabel.style.color = "black"; var errorCheck = doc.querySelector('td[class="error_title"]'); if (doc.title.search(/Status$/) > 0) // status overview parseStatus(); else if (doc.querySelector('td[colspan="4"]')) // valid status detail, but no data parseMisc("next"); else if (doc.title.search(/Status Detail/) > 0) // status detail with data parseDetail(); else if (errorCheck) { // encountered an error page // hit max request rate if (~errorCheck.textContent.indexOf("page request rate")) { var _d = doc.documentURI.match(/\d{8}/)[0], _p = doc.documentURI.match(/ber=(\d+)/)[1]; metrics.dbupdate.mark("[PRE]"+_d+"p"+_p, "start"); console.log("exceeded max requests; refetching", doc.documentURI); statusLabel.innerHTML = "Exceeded maximum server requests; retrying "+HITStorage.ISODate(_d)+" page "+_p+"."+ "
Please wait..."; setTimeout(HITStorage.fetch, 550, doc.documentURI); return; } // no more staus details left in range else if (qc.extraDays) parseMisc("end"); } else throw "ParseError::unhandled document received @"+doc.documentURI; function parseStatus() {//{{{ HITStorage.data = { HIT: [], STATS: [] }; qc.seen = {}; ProjectedEarnings.clear(); var _pastDataExists = Boolean(Object.keys(qc.fetchData).length); var raw = { day: doc.querySelectorAll(".statusDateColumnValue"), sub: doc.querySelectorAll(".statusSubmittedColumnValue"), app: doc.querySelectorAll(".statusApprovedColumnValue"), rej: doc.querySelectorAll(".statusRejectedColumnValue"), pen: doc.querySelectorAll(".statusPendingColumnValue"), pay: doc.querySelectorAll(".statusEarningsColumnValue") }; var timeout = 0; for (var i=0;i qc.fetchData.lastDate) || ~(Object.keys(qc.fetchData).indexOf(_date)) ) { setTimeout(HITStorage.fetch, timeout, MTURK_BASE+"statusdetail", payload); timeout += 250; qc.fetchData[_date] = { submitted: d.submitted, pending: d.pending }; } } else { // get everything setTimeout(HITStorage.fetch, timeout, MTURK_BASE+"statusdetail", payload); timeout += 250; qc.fetchData[_date] = { submitted: d.submitted, pending: d.pending }; } } // for qc.fetchData.expectedTotal = _calcTotals(qc.fetchData); // try for extra days if (qc.extraDays === true) { localStorage.removeItem("hitdb_extraDays"); d = _decDate(HITStorage.data.STATS[HITStorage.data.STATS.length-1].date); qc.extraDays = d; // repurpose extraDays for QC payload = { encodedDate: d, pageNumber: 1, sortType: "All" }; console.log("fetchrequest for", d, "sent by parseStatus"); setTimeout(HITStorage.fetch, 1000, MTURK_BASE+"statusdetail", payload); } qc.fetchData.lastDate = HITStorage.data.STATS[0].date; // most recent date seen }//}}} parseStatus function parseDetail() {//{{{ var _date = doc.documentURI.replace(/.+(\d{8}).+/, "$1"); var _page = doc.documentURI.replace(/.+ber=(\d+).+/, "$1"); metrics.dbupdate.mark("[PRE]"+_date+"p"+_page, "end"); console.log("page:", _page, "date:", _date); statusLabel.textContent = "Processing "+HITStorage.ISODate(_date)+" page "+_page; var raw = { req: doc.querySelectorAll(".statusdetailRequesterColumnValue"), title: doc.querySelectorAll(".statusdetailTitleColumnValue"), pay: doc.querySelectorAll(".statusdetailAmountColumnValue"), status: doc.querySelectorAll(".statusdetailStatusColumnValue"), feedback: doc.querySelectorAll(".statusdetailRequesterFeedbackColumnValue") }; for (var i=0;i 1) { setTimeout(HITStorage.fetch, 250, MTURK_BASE+"statusdetail", payload); console.log("going to next page", payload.encodedDate); } else if (type === "end" && +qc.extraDays > 1) { statusLabel.textContent = "Writing to database..."; HITStorage.write(HITStorage.data, "update"); } else throw 'Unhandled case -- "'+type+'" in '+doc.documentURI; }//}}} function _decDate(date) {//{{{ var y = date.substr(0,4); var m = date.substr(5,2); var d = date.substr(8,2); date = new Date(y,m-1,d-1); return Number(date.getMonth()+1).toPadded() + Number(date.getDate()).toPadded() + date.getFullYear(); }//}}} function _calcTotals(obj) {//{{{ var sum = 0; for (var k in obj){ if (obj.hasOwnProperty(k) && !isNaN(+k)) sum += obj[k].submitted; } return sum; }//}}} },//}}} parseDOM ISODate: function(date) { //{{{ MMDDYYYY <-> YYYY-MM-DD 'use strict'; if (date.length === 10) return date.substr(5,2)+date.substr(-2)+date.substr(0,4); else return date.substr(4)+"-"+date.substr(0,2)+"-"+date.substr(2,2); }, //}}} ISODate fetch: function(url, payload) { //{{{ 'use strict'; //format GET request with query payload if (payload) { var args = 0; url += "?"; for (var k in payload) { if (payload.hasOwnProperty(k)) { if (args++) url += "&"; url += k + "=" + payload[k]; } } } // defer XHR to a promise var fetch = new Promise( function(fulfill, deny) { var urlreq = new XMLHttpRequest(); urlreq.open("GET", url, true); urlreq.responseType = "document"; urlreq.send(); urlreq.onload = function() { if (this.status === 200) { fulfill(this.response); } else { deny("Error ".concat(String(this.status)).concat(": "+this.statusText)); } }; urlreq.onerror = function() { deny("Error ".concat(String(this.status)).concat(": "+this.statusText)); }; urlreq.ontimeout = function() { deny("Error ".concat(String(this.status)).concat(": "+this.statusText)); }; } ); fetch.then( HITStorage.parseDOM, HITStorage.error ); }, //}}} fetch write: function(input, statusUpdate) { //{{{ 'use strict'; if (statusUpdate === "update") qc.timeoutTimer = setTimeout(HITStorage.error, 5555, "DatabaseCreationError"); var dbh = window.indexedDB.open(DB_NAME); dbh.onerror = HITStorage.error; dbh.onsuccess = function() { _write(this.result); }; var counts = { requests: 0, total: 0 }; function _write(db) { db.onerror = HITStorage.error; var os = Object.keys(input); var dbt = db.transaction(os, "readwrite"); var dbo = []; for (var i=0;i= this.data.weekEnd) || (!isToday && _date.getDay() < this.data.day) ) { // new week this.data.earnings.week = 0; this.data.weekEnd = weekEnd; this.data.weekStart = weekStart; } if (date !== this.data.today || !isToday) { // new day this.data.today = date; this.data.day = _date.getDay(); this.data.earnings.day = 0; } this.saveState(); },//}}} updateDate draw: function(init) {//{{{ 'use strict'; var parentTable = document.querySelector("#total_earnings_amount").offsetParent, rowPending = init ? parentTable.insertRow(-1) : parentTable.rows[4], rowProjectedDay = init ? parentTable.insertRow(-1) : parentTable.rows[5], rowProjectedWeek = init ? parentTable.insertRow(-1) : parentTable.rows[6], title = "Click to set/change the target value"; if (init) { rowPending.insertCell(-1);rowPending.insertCell(-1);rowPending.className = "even"; rowProjectedDay.insertCell(-1);rowProjectedDay.insertCell(-1);rowProjectedDay.className = "odd"; rowProjectedWeek.insertCell(-1);rowProjectedWeek.insertCell(-1);rowProjectedWeek.className = "even"; for (var i=0;i[ ' + this.data.dbUpdated + ' ]'; rowPending.cells[1].textContent = "$"+Number(this.data.pending).toFixed(2); rowProjectedDay.cells[0].innerHTML = 'Projected earnings for the day
'+ ''+ ' ' + Number(this.data.earnings.day-this.data.target.day).toFixed(2) + ''; rowProjectedDay.cells[1].textContent = "$"+Number(this.data.earnings.day).toFixed(2); rowProjectedWeek.cells[0].innerHTML = 'Projected earnings for the week
' + '' + ' ' + Number(this.data.earnings.week-this.data.target.week).toFixed(2) + ''; rowProjectedWeek.cells[1].textContent = "$"+Number(this.data.earnings.week).toFixed(2); document.querySelector("#projectedDayProgress").onclick = updateTargets.bind(this, "day"); document.querySelector("#projectedWeekProgress").onclick = updateTargets.bind(this, "week"); function updateTargets(span, e) { /*jshint validthis:true*/ var goal = prompt("Set your " + (span === "day" ? "daily" : "weekly") + " target:", this.data.target[span === "day" ? "day" : "week"]); if (goal && !isNaN(goal)) { this.data.target[span === "day" ? "day" : "week"] = goal; e.target.max = goal; e.target.nextSibling.textContent = Number(this.data.earnings[span==="day" ? "day":"week"] - goal).toFixed(2); this.saveState(); } } },//}}} draw saveState: function() { 'use strict'; localStorage.setItem("hitdb_projectedEarnings", JSON.stringify(this.data)); }, clear: function() { 'use strict'; this.data.earnings = { day:0, week:0 }; this.data.pending = 0; }, updateValues: function(obj) { 'use strict'; var vDate = Date.parse(obj.date); if (~obj.status.search(/pending/i)) // sum pending earnings (include approved until fully cleared as paid) this.data.pending = Math.decRound(obj.reward+this.data.pending, 2); if (HITStorage.ISODate(obj.date) === this.data.today && !~obj.status.search(/rejected/i)) // sum daily earnings this.data.earnings.day = Math.decRound(obj.reward+this.data.earnings.day, 2); if (vDate < this.data.weekEnd && vDate >= this.data.weekStart && !~obj.status.search(/rejected/i)) // sum weekly earnings this.data.earnings.week = Math.decRound(obj.reward+this.data.earnings.week, 2); } };//}}} ProjectedEarnings function DatabaseResult() {//{{{ 'use strict'; this.results = []; this.formatHTML = function(type, simple) {//{{{ simple = simple || false; var count = 0, htmlTxt = [], entry = null, _trClass = null; if (this.results.length < 1) return "

No entries found matching your query.

"; if (type === "daily") { htmlTxt.push(''+ 'DateSubmittedApprovedRejectedPendingEarnings'); var r = _collate(this.results,"date"); for (entry of this.results) { _trClass = (count++ % 2 === 0) ? 'class="even"' : 'class="odd"'; htmlTxt.push(''+ '' + entry.date + '' + entry.submitted + '' + '' + entry.approved + '' + entry.rejected + '' + entry.pending + '' + '' + Number(entry.earnings).toFixed(2) + ''); } htmlTxt.push('Totals:' + '' + r.totalEntries + ' days' + r.totalSub + '' + '' + r.totalApp + '' + r.totalRej + '' + '' + r.totalPen + '$' + Number(Math.decRound(r.totalPay,2)).toFixed(2) + ''); } else if (type === "pending" || type === "requester") { htmlTxt.push('Requester ID' + 'Requester' + (type === "pending" ? 'Pending' : 'HITs') + 'Rewards'); r = _collate(this.results,"requesterId"); for (var k in r) { if (!~k.search(/total/) && r.hasOwnProperty(k)) { var tr = ['' + '' + '[+] ' + r[k][0].requesterId + '' + r[k][0].requesterName + '' + '' + r[k].length + '' + Number(Math.decRound(r[k].pay,2)).toFixed(2) + '']; for (var hit of r[k]) { // hits in range per requester id tr.push('' + hit.date + '' + '' + hit.title + '' + _parseRewards(hit.reward,"pay") + ''); } htmlTxt.push(tr.join('')); } } htmlTxt.sort(function(a,b) { return +b.substr(15,5).match(/\d+/) - +a.substr(15,5).match(/\d+/); }); htmlTxt.push('Totals:' + '' + (Object.keys(r).length-3) + ' Requesters' + '' + r.totalEntries + ''+ '$' + Number(Math.decRound(r.totalPay,2)).toFixed(2) + ''); } else { // default if (!simple) htmlTxt.push('' + 'Reward'+ '' + 'DateRequesterHIT titlePay'+ 'BonusStatusFeedback'); for (entry of this.results) { _trClass = (count++ % 2 === 0) ? 'class="even"' : 'class="odd"'; var _stColor = ~entry.status.search(/(paid|approved)/i) ? "green" : entry.status === "Pending Approval" ? "orange" : "red"; var href = MTURK_BASE+'contact?requesterId='+entry.requesterId+'&requesterName='+entry.requesterName+ '&subject=Regarding+Amazon+Mechanical+Turk+HIT+'+entry.hitId; if (!simple) htmlTxt.push(''+ '' + entry.date + '' + '' + entry.requesterName + '' + '' + ' 📝 ' + entry.title + '' + _parseRewards(entry.reward,"pay") + '' + '' + (+_parseRewards(entry.reward,"bonus") ? _parseRewards(entry.reward,"bonus") : "") + '' + entry.status + '' + entry.feedback + ''); else htmlTxt.push(''+entry.date+''+entry.title+''+ _parseRewards(entry.reward,"pay") + ''+ entry.status+''); } if (!simple) { r = _collate(this.results,"requesterId"); htmlTxt.push('' + 'Totals:' + r.totalEntries + ' HITs' + '$' + Number(Math.decRound(r.totalPay,2)).toFixed(2) + '' + '$' + Number(Math.decRound(r.totalBonus,2)).toFixed(2) + '' + ''); } } return htmlTxt.join(''); };//}}} formatHTML this.formatCSV = function(type) {//{{{ var csvTxt = [], entry = null, delimiter="\t"; if (type === "daily") { csvTxt.push( ["Date", "Submitted", "Approved", "Rejected", "Pending", "Earnings\n"].join(delimiter) ); for (entry of this.results) { csvTxt.push( [entry.date, entry.submitted, entry.approved, entry.rejected, entry.pending, Number(entry.earnings).toFixed(2)+"\n"].join(delimiter) ); } csvToFile(csvTxt, "hitdb_dailyOverview.csv"); } else if (type === "pending" || type === "requester") { csvTxt.push( ["RequesterId","Requester", (type === "pending" ? "Pending" : "HITs"), "Rewards\n"].join(delimiter) ); var r = _collate(this.results,"requesterId"); for (var k in r) { if (!~k.search(/total/) && r.hasOwnProperty(k)) csvTxt.push( [k, r[k][0].requesterName, r[k].length, Number(Math.decRound(r[k].pay,2)).toFixed(2)+"\n"].join(delimiter) ); } csvToFile(csvTxt, "hitdb_"+type+"Overview.csv"); } else { csvTxt.push(["Date","Requester","Title","Pay","Bonus","Status","Feedback\n"].join(delimiter)); for (entry of this.results) { csvTxt.push([entry.date, entry.requesterName, entry.title, Number(_parseRewards(entry.reward,"pay")).toFixed(2), (+_parseRewards(entry.reward,"bonus") ? Number(_parseRewards(entry.reward,"bonus")).toFixed(2) : ""), entry.status, entry.feedback+"\n"].join(delimiter)); } csvToFile(csvTxt, "hitdb_queryResults.csv"); } return "
"+csvTxt.join('')+"
"; function csvToFile(csv, filename) { var blob = new Blob(csv, {type: "text/csv", endings: "native"}), dl = document.createElement("A"); dl.href = URL.createObjectURL(blob); dl.download = filename; document.body.appendChild(dl); // FF doesn't support forced events unless element is part of the document dl.click(); // so we make it so and click, dl.remove(); // then immediately remove it return dl; } };//}}} formatCSV this.include = function(value) { this.results.push(value); }; function _parseRewards(rewards,value) { if (!isNaN(rewards)) { if (value === "pay") return Number(rewards).toFixed(2); else return "0.00"; } else { if (value === "pay") return Number(rewards.pay).toFixed(2); else return Number(rewards.bonus).toFixed(2); } } // _parse function _collate(data, index) { var r = { totalPay: 0, totalBonus: 0, totalEntries: data.length, totalSub: 0, totalApp: 0, totalRej: 0, totalPen: 0 }; for (var e of data) { if (!r[e[index]]) { r[e[index]] = []; r[e[index]].pay = 0; } r[e[index]].push(e); if (index === "date") { r.totalSub += e.submitted; r.totalApp += e.approved; r.totalRej += e.rejected; r.totalPen += e.pending; r.totalPay += e.earnings; } else { r[e[index]].pay += (+_parseRewards(e.reward,"pay")); r.totalPay += (+_parseRewards(e.reward,"pay")); r.totalBonus += (+_parseRewards(e.reward,"bonus")); } } return r; } // _collate }//}}} databaseresult /* * * Above contains the core functions. Below is the * main body, interface, and tangential functions. * *///{{{ // the Set() constructor is never actually used other than to test for Chrome v38+ if (!("indexedDB" in window && "Set" in window)) alert("HITDB::Your browser is too outdated or otherwise incompatible with this script!"); else { /* var tdbh = window.indexedDB.open(DB_NAME); tdbh.onerror = function(e) { 'use strict'; console.log("[TESTDB]",e.target.error.name+":", e.target.error.message, e); }; tdbh.onsuccess = INFLATEDUMMYVALUES; tdbh.onupgradeneeded = BLANKSLATE; var dbh = null; */ if (document.location.pathname.search(/dashboard/) > 0) { var dbh = window.indexedDB.open(DB_NAME, DB_VERSION); dbh.onerror = function(e) { 'use strict'; console.log("[HITDB]",e.target.error.name+":", e.target.error.message, e); }; dbh.onupgradeneeded = HITStorage.versionChange; dbh.onsuccess = function() { 'use strict'; this.result.close(); }; dashboardUI(); ProjectedEarnings.updateDate(); ProjectedEarnings.draw(true); } else beenThereDoneThat(); } /*}}} * * Above is the main body and core functions. Below * defines UI layout/appearance and tangential functions. * */ // {{{ css injection var css = ""; document.head.innerHTML += css; // }}} function beenThereDoneThat() {//{{{ // // TODO refine searching // 'use strict'; var qualNode = document.querySelector('td[colspan="11"]'); if (qualNode) { // we're on the preview page! var requester = document.querySelector('input[name="requesterId"]').value, //hitId = document.querySelector('input[name="hitId"]').value, autoApproval = document.querySelector('input[name="hitAutoAppDelayInSeconds"]').value, hitTitle = document.querySelector('div[style*="ellipsis"]').textContent.trim().replace(/\|/g,""), insertionNode = qualNode.parentNode.parentNode; var row = document.createElement("TR"), cellL = document.createElement("TD"), cellR = document.createElement("TD"); var _resultsTable = document.createElement("TABLE"); _resultsTable.id = "resultsTableFor"+requester; insertionNode.parentNode.parentNode.appendChild(_resultsTable); cellR.innerHTML = 'Auto-Approval:  '+_ftime(autoApproval); var rbutton = document.createElement("BUTTON"); rbutton.classList.add("hitdbRTButtons","hitdbRTButtons-large"); rbutton.textContent = "Requester"; rbutton.onclick = function(e) { e.preventDefault(); showResults(requester); }; var tbutton = rbutton.cloneNode(false); rbutton.dataset.id = requester; rbutton.title = "Show HITs completed from this requester"; tbutton.textContent = "HIT Title"; tbutton.onclick = function(e) { e.preventDefault(); }; HITStorage.recall("HIT", {index: "requesterId", range: window.IDBKeyRange.only(requester)}) .then(processResults.bind(rbutton)); HITStorage.recall("HIT", {index: "title", range: window.IDBKeyRange.only(hitTitle)}) .then(processResults.bind(tbutton)); row.appendChild(cellL); row.appendChild(cellR); cellL.appendChild(rbutton); cellL.appendChild(tbutton); cellL.colSpan = "3"; cellR.colSpan = "8"; insertionNode.appendChild(row); } else { // browsing HITs n sutff var titleNodes = document.querySelectorAll('a[class="capsulelink"]'); if (titleNodes.length < 1) return; // nothing left to do here! var requesterNodes = document.querySelectorAll('a[href*="hitgroups&requester"]'); var insertionNodes = []; for (var i=0;i0) ? d+" day"+(d>1 ? "s " : " ") : "") + ((h>0) ? h+"h " : "") + ((m>0) ? m+"m " : "") + ((s>0) ? s+"s" : ""); } }//}}} btdt function dashboardUI() {//{{{ // // TODO refactor // 'use strict'; var controlPanel = document.createElement("TABLE"); var insertionNode = document.querySelector(".footer_separator").previousSibling; document.body.insertBefore(controlPanel, insertionNode); controlPanel.width = "760"; controlPanel.align = "center"; controlPanel.cellSpacing = "0"; controlPanel.cellPadding = "0"; controlPanel.innerHTML = '' + '' + 'HIT Database Mk. II ' + '(What\'s this?)' + '' + '
' + '' + '' + '' + '' + '
' + '' + '' + '' + '
' + '' + '' + '' + '' + '
' + '' + '' + '' + '' + '
' + '' + '
' + '
'; var updateBtn = document.querySelector("#hdbUpdate"), backupBtn = document.querySelector("#hdbBackup"), restoreBtn = document.querySelector("#hdbRestore"), fileInput = document.querySelector("#hdbFileInput"), exportCSVInput = document.querySelector("#hdbCSVInput"), searchBtn = document.querySelector("#hdbSearch"), searchInput = document.querySelector("#hdbSearchInput"), pendingBtn = document.querySelector("#hdbPending"), reqBtn = document.querySelector("#hdbRequester"), dailyBtn = document.querySelector("#hdbDaily"), fromdate = document.querySelector("#hdbMinDate"), todate = document.querySelector("#hdbMaxDate"), statusSelect = document.querySelector("#hdbStatusSelect"), progressBar = document.querySelector("#hdbProgressBar"); var searchResults = document.createElement("DIV"); searchResults.align = "center"; searchResults.id = "hdbSearchResults"; searchResults.style.display = "block"; searchResults.innerHTML = '[ clear results ]
' + '
'; document.body.insertBefore(searchResults, insertionNode); searchResults.firstChild.onclick = function(e) { e.target.style.display = "none"; searchResults.children[2].innerHTML = null; }; updateBtn.onclick = function() { progressBar.style.display = "block"; metrics.dbupdate = new Metrics("database_update"); HITStorage.fetch(MTURK_BASE+"status"); document.querySelector("#hdbStatusText").textContent = "fetching status page...."; }; exportCSVInput.addEventListener("click", function() { if (exportCSVInput.checked) { searchBtn.textContent = "Export CSV"; pendingBtn.textContent += " (csv)"; reqBtn.textContent += " (csv)"; dailyBtn.textContent += " (csv)"; } else { searchBtn.textContent = "Search"; pendingBtn.textContent = pendingBtn.textContent.replace(" (csv)",""); reqBtn.textContent = reqBtn.textContent.replace(" (csv)",""); dailyBtn.textContent = dailyBtn.textContent.replace(" (csv)", ""); } }); fromdate.addEventListener("focus", function() { var offsets = getPosition(this, true); new Calendar(offsets.x, offsets.y, this).drawCalendar(); }); todate.addEventListener("focus", function() { var offsets = getPosition(this, true); new Calendar(offsets.x, offsets.y, this).drawCalendar(); }); backupBtn.onclick = HITStorage.backup; restoreBtn.onclick = function() { fileInput.click(); }; fileInput.onchange = processFile; searchBtn.onclick = function() { var r = getRange(); var _filter = { status: statusSelect.value, query: searchInput.value.trim().length > 0 ? searchInput.value : "*" }; var _opt = { index: "date", range: r.range, dir: r.dir, filter: _filter, progress: true }; metrics.dbrecall = new Metrics("database_recall::search"); HITStorage.recall("HIT", _opt).then(function(r) { searchResults.children[0].style.display = "initial"; searchResults.children[2].innerHTML = exportCSVInput.checked ? r.formatCSV() : r.formatHTML(); autoScroll("#hdbSearchResults"); for (var _r of r.results) { // retrieve and append notes HITStorage.recall("NOTES", { index: "hitId", range: window.IDBKeyRange.only(_r.hitId) }).then(noteHandler.bind(null,"attach")); } var el = null; for (el of document.querySelectorAll(".bonusCell")) { el.dataset.initial = el.textContent; el.onblur = updateBonus; el.onkeydown = updateBonus; } for (el of document.querySelectorAll('span[id^="note-"]')) { el.onclick = noteHandler.bind(null,"new"); } metrics.dbrecall.stop(); metrics.dbrecall.report(); progressBar.style.display = "none"; }); }; // search button click event pendingBtn.onclick = function() { var r = getRange(); var _filter = { status: "Approval", query: searchInput.value.trim().length > 0 ? searchInput.value : "*" }, _opt = { index: "date", dir: "prev", range: r.range, filter: _filter, progress: true }; metrics.dbrecall = new Metrics("database_recall::pending"); HITStorage.recall("HIT", _opt).then(function(r) { searchResults.children[0].style.display = "initial"; searchResults.children[2].innerHTML = exportCSVInput.checked ? r.formatCSV("pending") : r.formatHTML("pending"); autoScroll("#hdbSearchResults"); var expands = document.querySelectorAll(".hdbExpandRow"); for (var el of expands) { el.onclick = showHiddenRows; } metrics.dbrecall.stop(); metrics.dbrecall.report(); progressBar.style.display = "none"; }); }; //pending overview click event reqBtn.onclick = function() { var r = getRange(); var _opt = { index: "date", range: r.range, progress: true }; metrics.dbrecall = new Metrics("database_recall::requester"); HITStorage.recall("HIT", _opt).then(function(r) { searchResults.children[0].style.display = "initial"; searchResults.children[2].innerHTML = exportCSVInput.checked ? r.formatCSV("requester") : r.formatHTML("requester"); autoScroll("#hdbSearchResults"); var expands = document.querySelectorAll(".hdbExpandRow"); for (var el of expands) { el.onclick = showHiddenRows; } metrics.dbrecall.stop(); metrics.dbrecall.report(); progressBar.style.display = "none"; }); }; //requester overview click event dailyBtn.onclick = function() { metrics.dbrecall = new Metrics("database_recall::daily"); HITStorage.recall("STATS", { dir: "prev" }).then(function(r) { searchResults.children[0].style.display = "initial"; searchResults.children[2].innerHTML = exportCSVInput.checked ? r.formatCSV("daily") : r.formatHTML("daily"); autoScroll("#hdbSearchResults"); metrics.dbrecall.stop(); metrics.dbrecall.report(); }); }; //daily overview click event function getRange() { var _min = fromdate.value.length === 10 ? fromdate.value : undefined, _max = todate.value.length === 10 ? todate.value : undefined; var _range = (_min === undefined && _max === undefined) ? null : (_min === undefined) ? window.IDBKeyRange.upperBound(_max) : (_max === undefined) ? window.IDBKeyRange.lowerBound(_min) : (_max < _min) ? window.IDBKeyRange.bound(_max,_min) : window.IDBKeyRange.bound(_min,_max); return { min: _min, max: _max, range: _range, dir: _max < _min ? "prev" : "next" }; } function getPosition(element, includeHeight) { var offsets = { x: 0, y: includeHeight ? element.offsetHeight : 0 }; do { offsets.x += element.offsetLeft; offsets.y += element.offsetTop; element = element.offsetParent; } while (element); return offsets; } }//}}} dashboard function showHiddenRows(e) {//{{{ 'use strict'; var rid = e.target.parentNode.textContent.substr(4); var nodes = document.querySelectorAll('tr[data-rid="'+rid+'"]'), el = null; if (e.target.textContent === "[+]") { for (el of nodes) el.style.display="table-row"; e.target.textContent = "[-]"; } else { for (el of nodes) el.style.display="none"; e.target.textContent = "[+]"; } }//}}} function updateBonus(e) {//{{{ 'use strict'; if (e instanceof window.KeyboardEvent && e.keyCode === 13) { e.target.blur(); return false; } else if (e instanceof window.FocusEvent) { var _bonus = +e.target.textContent.replace(/\$/,""); if (_bonus !== +e.target.dataset.initial) { console.log("updating bonus to",_bonus,"from",e.target.dataset.initial,"("+e.target.dataset.hitid+")"); e.target.dataset.initial = _bonus; var _pay = +e.target.previousSibling.textContent, _range = window.IDBKeyRange.only(e.target.dataset.hitid); window.indexedDB.open(DB_NAME).onsuccess = function() { this.result.transaction("HIT", "readwrite").objectStore("HIT").openCursor(_range).onsuccess = function() { var c = this.result; if (c) { var v = c.value; v.reward = { pay: _pay, bonus: _bonus }; c.update(v); } }; // idbcursor }; // idbopen } // bonus is new value } // keycode } //}}} updateBonus function noteHandler(type, e) {//{{{ // // TODO restructure event handling/logic tree // combine save and delete; it's ugly :( // actually this whole thing is messy and in need of refactoring // 'use strict'; if (e instanceof window.KeyboardEvent) { if (e.keyCode === 13) { e.target.blur(); return false; } return; } if (e instanceof window.FocusEvent) { if (e.target.textContent.trim() !== e.target.dataset.initial) { if (!e.target.textContent.trim()) { e.target.previousSibling.previousSibling.firstChild.click(); return; } var note = e.target.textContent.trim(), _range = window.IDBKeyRange.only(e.target.dataset.id), inote = e.target.dataset.initial, hitId = e.target.dataset.id, date = e.target.previousSibling.textContent; e.target.dataset.initial = note; window.indexedDB.open(DB_NAME).onsuccess = function() { this.result.transaction("NOTES", "readwrite").objectStore("NOTES").index("hitId").openCursor(_range).onsuccess = function() { if (this.result) { var r = this.result.value; if (r.note === inote) { // note already exists in database, so we update its value r.note = note; this.result.update(r); return; } this.result.continue(); } else { if (this.source instanceof window.IDBObjectStore) this.source.put({ note:note, date:date, hitId:hitId }); else this.source.objectStore.put({ note:note, date:date, hitId:hitId }); } }; this.result.close(); }; } return; // end of save event; no need to proceed } if (type === "delete") { var tr = e.target.parentNode.parentNode, noteCell = tr.lastChild; _range = window.IDBKeyRange.only(noteCell.dataset.id); if (!noteCell.dataset.initial) tr.remove(); else { window.indexedDB.open(DB_NAME).onsuccess = function() { this.result.transaction("NOTES", "readwrite").objectStore("NOTES").index("hitId").openCursor(_range).onsuccess = function() { if (this.result) { if (this.result.value.note === noteCell.dataset.initial) { this.result.delete(); tr.remove(); return; } this.result.continue(); } }; this.result.close(); }; } return; // end of deletion event; no need to proceed } else { if (type === "attach" && !e.results.length) return; var trow = e instanceof window.MouseEvent ? e.target.parentNode.parentNode : null, tbody = trow ? trow.parentNode : null, row = document.createElement("TR"), c1 = row.insertCell(0), c2 = row.insertCell(1), c3 = row.insertCell(2); date = new Date(); hitId = e instanceof window.MouseEvent ? e.target.id.substr(5) : null; c1.innerHTML = '[x]'; c1.firstChild.onclick = noteHandler.bind(null,"delete"); c1.style.textAlign = "right"; c2.title = "Date on which the note was added"; c3.style.color = "crimson"; c3.colSpan = "5"; c3.contentEditable = "true"; c3.onblur = noteHandler.bind(null,"blur"); c3.onkeydown = noteHandler.bind(null, "kb"); if (type === "new") { row.classList.add(trow.classList); tbody.insertBefore(row, trow.nextSibling); c2.textContent = date.getFullYear()+"-"+Number(date.getMonth()+1).toPadded()+"-"+Number(date.getDate()).toPadded(); c3.dataset.initial = ""; c3.dataset.id = hitId; c3.focus(); return; } for (var entry of e.results) { trow = document.querySelector('tr[data-id="'+entry.hitId+'"]'); tbody = trow.parentNode; row = row.cloneNode(true); c1 = row.firstChild; c2 = c1.nextSibling; c3 = row.lastChild; row.classList.add(trow.classList); tbody.insertBefore(row, trow.nextSibling); c1.firstChild.onclick = noteHandler.bind(null,"delete"); c2.textContent = entry.date; c3.textContent = entry.note; c3.dataset.initial = entry.note; c3.dataset.id = entry.hitId; c3.onblur = noteHandler.bind(null,"blur"); c3.onkeydown = noteHandler.bind(null, "kb"); } } // new/attach }//}}} noteHandler function processFile(e) {//{{{ 'use strict'; var f = e.target.files; if (f.length && f[0].name.search(/\.bak$/) && ~f[0].type.search(/(text|json)/)) { var reader = new FileReader(), testing = true; reader.readAsText(f[0].slice(0,10)); reader.onload = function(e) { if (testing && e.target.result.search(/(STATS|NOTES|HIT)/) < 0) { return error(); } else if (testing) { testing = false; document.querySelector("#hdbProgressBar").style.display = "block"; reader.readAsText(f[0]); } else { var data = JSON.parse(e.target.result); console.log(data); HITStorage.write(data, "restore"); } }; // reader.onload } else { error(); } function error() { var s = document.querySelector("#hdbStatusText"), e = "Restore::FileReadError : encountered unsupported file"; s.style.color = "red"; s.textContent = e; throw e; } }//}}} processFile function autoScroll(location, dt) {//{{{ 'use strict'; var target = document.querySelector(location).offsetTop, pos = window.scrollY, dpos = Math.ceil((target - pos)/3); dt = dt ? dt-1 : 25; // time step/max recursions if (target === pos || dpos === 0 || dt === 0) return; window.scrollBy(0, dpos); setTimeout(function() { autoScroll(location, dt); }, dt); }//}}} function Calendar(offsetX, offsetY, caller) {//{{{ 'use strict'; this.date = new Date(); this.offsetX = offsetX; this.offsetY = offsetY; this.caller = caller; this.drawCalendar = function(year,month,day) {//{{{ year = year || this.date.getFullYear(); month = month || this.date.getMonth()+1; day = day || this.date.getDate(); var longMonths = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; var date = new Date(year,month-1,day); var anchors = _getAnchors(date); //make new container if one doesn't already exist var container = null; if (document.querySelector("#hdbCalendarPanel")) { container = document.querySelector("#hdbCalendarPanel"); container.removeChild( container.getElementsByTagName("TABLE")[0] ); } else { container = document.createElement("DIV"); container.id = "hdbCalendarPanel"; document.body.appendChild(container); } container.style.left = this.offsetX; container.style.top = this.offsetY; var cal = document.createElement("TABLE"); cal.cellSpacing = "0"; cal.cellPadding = "0"; cal.border = "0"; container.appendChild(cal); cal.innerHTML = '' + '<' + '' + ''+date.getFullYear()+'
'+longMonths[date.getMonth()]+'' + '' + '>' + 'SM' + 'TWT' + 'FS'; document.querySelector('th[title="Previous month"]').addEventListener( "click", function() { this.drawCalendar(date.getFullYear(), date.getMonth(), 1); }.bind(this) ); document.querySelector('th[title="Previous year"]').addEventListener( "click", function() { this.drawCalendar(date.getFullYear()-1, date.getMonth()+1, 1); }.bind(this) ); document.querySelector('th[title="Next month"]').addEventListener( "click", function() { this.drawCalendar(date.getFullYear(), date.getMonth()+2, 1); }.bind(this) ); document.querySelector('th[title="Next year"]').addEventListener( "click", function() { this.drawCalendar(date.getFullYear()+1, date.getMonth()+1, 1); }.bind(this) ); var hasDay = false, thisDay = 1; for (var i=0;i<6;i++) { // cycle weeks var row = document.createElement("TR"); for (var j=0;j<7;j++) { // cycle days if (!hasDay && j === anchors.first && thisDay < anchors.total) hasDay = true; else if (hasDay && thisDay > anchors.total) hasDay = false; var cell = document.createElement("TD"); cell.classList.add("hdbCalCells"); row.appendChild(cell); if (hasDay) { cell.classList.add("hdbCalDays"); cell.textContent = thisDay; cell.addEventListener("click", _clickHandler.bind(this)); cell.dataset.year = date.getFullYear(); cell.dataset.month = date.getMonth()+1; cell.dataset.day = thisDay++; } } // for j cal.appendChild(row); } // for i function _clickHandler(e) { /*jshint validthis:true*/ var y = e.target.dataset.year; var m = Number(e.target.dataset.month).toPadded(); var d = Number(e.target.dataset.day).toPadded(); this.caller.value = y+"-"+m+"-"+d; this.die(); } function _getAnchors(date) { var _anchors = {}; date.setMonth(date.getMonth()+1); date.setDate(0); _anchors.total = date.getDate(); date.setDate(1); _anchors.first = date.getDay(); return _anchors; } };//}}} drawCalendar this.die = function() { document.querySelector("#hdbCalendarPanel").remove(); }; }//}}} Calendar // instance metrics apart from window scoped PerformanceTiming API function Metrics(name) {//{{{ 'use strict'; this.name = name || "undefined"; this.marks = {}; this.start = window.performance.now(); this.end = null; this.stop = function(){ if (!this.end) this.end = window.performance.now(); else throw "Metrics::AccessViolation: end point cannot be overwritten"; }; this.mark = function(name,position) { if (position === "end" && (!this.marks[name] || this.marks[name].end)) return; if (!this.marks[name]) this.marks[name] = {}; this.marks[name][position] = window.performance.now(); }; this.report = function() { console.group("Metrics for",this.name.toUpperCase()); console.log("Process completed in",+Number((this.end-this.start)/1000).toFixed(3),"seconds"); for (var k in this.marks) { if (this.marks.hasOwnProperty(k)) { console.log(k,"occurred after",+Number((this.marks[k].start-this.start)/1000).toFixed(3),"seconds,", "resolving in", +Number((this.marks[k].end-this.marks[k].start)/1000).toFixed(3), "seconds"); } } console.groupEnd(); }; }//}}} /* * * * * * * * * * * * * * * * TESTING FUNCTIONS -- DELETE BEFORE FINAL RELEASE * * * * * * * * * * * * * */ function INFLATEDUMMYVALUES() { //{{{ 'use strict'; var tdb = this.result; tdb.onerror = function(e) { console.log("requesterror",e.target.error.name,e.target.error.message,e); }; tdb.onversionchange = function(e) { console.log("tdb received versionchange request", e); tdb.close(); }; //console.log(tdb.transaction("HIT").objectStore("HIT").indexNames.contains("date")); console.groupCollapsed("Populating test database"); var tdbt = {}; tdbt.trans = tdb.transaction(["HIT", "NOTES", "BLOCKS"], "readwrite"); tdbt.hit = tdbt.trans.objectStore("HIT"); tdbt.notes = tdbt.trans.objectStore("NOTES"); tdbt.blocks= tdbt.trans.objectStore("BLOCKS"); var filler = { notes:[], hit:[], blocks:[]}; for (var n=0;n<100000;n++) { filler.hit.push({ date: "2015-08-00", requesterName: "tReq"+(n+1), title: "Greatest Title Ever #"+(n+1), reward: Number((n+1)%(200/n)+(((n+1)%200)/100)).toFixed(2), status: "moo", requesterId: ("RRRRRRR"+n).substr(-7), hitId: ("HHHHHHH"+n).substr(-7) }); if (n%1000 === 0) { filler.notes.push({ requesterId: ("RRRRRRR"+n).substr(-7), note: n+1 + " Proin vel erat commodo mi interdum rhoncus. Sed lobortis porttitor arcu, et tristique ipsum semper a." + " Donec eget aliquet lectus, vel scelerisque ligula." }); filler.blocks.push({requesterId: ("RRRRRRR"+n).substr(-7)}); } } _write(tdbt.hit, filler.hit); _write(tdbt.notes, filler.notes); _write(tdbt.blocks, filler.blocks); function _write(store, obj) { if (obj.length) { var t = obj.pop(); store.put(t).onsuccess = function() { _write(store, obj) }; } else { console.log("population complete"); } } console.groupEnd(); dbh = window.indexedDB.open(DB_NAME, DB_VERSION); dbh.onerror = function(e) { console.log("[HITDB]",e.target.error.name+":", e.target.error.message, e); }; console.log(dbh.readyState, dbh); dbh.onupgradeneeded = HITStorage.versionChange; dbh.onblocked = function(e) { console.log("blocked event triggered:", e); }; tdb.close(); }//}}} function BLANKSLATE() { //{{{ create empty db equivalent to original schema to test upgrade 'use strict'; var tdb = this.result; if (!tdb.objectStoreNames.contains("HIT")) { console.log("creating HIT OS"); var dbo = tdb.createObjectStore("HIT", { keyPath: "hitId" }); dbo.createIndex("date", "date", { unique: false }); dbo.createIndex("requesterName", "requesterName", { unique: false}); dbo.createIndex("title", "title", { unique: false }); dbo.createIndex("reward", "reward", { unique: false }); dbo.createIndex("status", "status", { unique: false }); dbo.createIndex("requesterId", "requesterId", { unique: false }); } if (!tdb.objectStoreNames.contains("STATS")) { console.log("creating STATS OS"); dbo = tdb.createObjectStore("STATS", { keyPath: "date" }); } if (!tdb.objectStoreNames.contains("NOTES")) { console.log("creating NOTES OS"); dbo = tdb.createObjectStore("NOTES", { keyPath: "requesterId" }); } if (!tdb.objectStoreNames.contains("BLOCKS")) { console.log("creating BLOCKS OS"); dbo = tdb.createObjectStore("BLOCKS", { keyPath: "id", autoIncrement: true }); dbo.createIndex("requesterId", "requesterId", { unique: false }); } } //}}} // vim: ts=2:sw=2:et:fdm=marker:noai