// ==UserScript== // @name [RED] Similar CD detector // @namespace https://greasyfork.org/users/321857-anakunda // @version 1.00 // @description Simple script for testing CD releases for duplicity // @match https://redacted.ch/torrents.php?id=* // @match https://redacted.ch/torrents.php?page=*&id=* // @run-at document-end // @author Anakunda // @copyright 2022, Anakunda (https://greasyfork.org/users/321857-anakunda) // @license GPL-3.0-or-later // @require https://openuserjs.org/src/libs/Anakunda/xhrLib.min.js // @downloadURL none // ==/UserScript== 'use strict'; const maxRemarks = 60, requestsCache = new Map; let selected = null; const getTorrentIds = (...trs) => trs.map(function(tr) { if (!(tr instanceof HTMLElement)) throw 'Invalid argument'; if ((tr = tr.querySelector('a.button_pl')) != null && (tr = parseInt(new URLSearchParams(tr.search).get('torrentid'))) > 0) return tr; throw 'Failed to get torrent id'; }); function testSimilarity(...torrentIds) { if (torrentIds.length < 2 || !torrentIds.every(torrentId => torrentId > 0)) return Promise.reject('Invalid argument'); const msfTime = '(?:(\\d+):)?(\\d+):(\\d+)[\\.\\:](\\d+)'; const msfToSector = time => Array.isArray(time) || (time = new RegExp('^\\s*' + msfTime + '\\s*$').exec(time)) != null ? (((time[1] ? parseInt(time[1]) : 0) * 60 + parseInt(time[2])) * 60 + parseInt(time[3])) * 75 + parseInt(time[4]) : NaN; const rxRR = /^(?:(?:Selected range)[^\S\r\n]+\((?:Sectors) (\d+)[^\S\r\n]*-[^\S\r\n]*(\d+)\))/m; const tocParser = '^\\s*' + ['\\d+', msfTime, msfTime, '\\d+', '\\d+'] .map(pattern => '(' + pattern + ')').join('\\s+\\|\\s+') + '\\s*$'; const tocEntriesMapper = tocEntries => Array.isArray(tocEntries) ? tocEntries.map(function(tocEntry, trackNdx) { if ((tocEntry = new RegExp(tocParser).exec(tocEntry)) == null) throw `assertion failed: track ${trackNdx + 1} ToC entry invalid format`; console.assert(msfToSector(tocEntry[2]) == parseInt(tocEntry[12])); console.assert(msfToSector(tocEntry[7]) == parseInt(tocEntry[13]) + 1 - parseInt(tocEntry[12])); return { trackNumber: parseInt(tocEntry[1]), startSector: parseInt(tocEntry[12]), endSector: parseInt(tocEntry[13]), }; }) : null; return Promise.all(torrentIds.map(function loadLogFile(torrentId) { if (requestsCache.has(torrentId)) return requestsCache.get(torrentId); const request = localXHR('/torrents.php?' + new URLSearchParams({ action: 'loglist', torrentid: torrentId })).then(document => Array.from(document.body.querySelectorAll(':scope > blockquote > pre:first-child'), pre => pre.textContent).filter(function(logfile) { const rr = rxRR.exec(logfile); if (rr == null) return true; // Ditch HTOA logs let tocEntries = logfile.match(new RegExp(tocParser, 'gm')); if (tocEntries != null) tocEntries = tocEntriesMapper(tocEntries); else return true; return parseInt(rr[1]) != 0 || parseInt(rr[2]) + 1 != tocEntries[0].startSector; })); requestsCache.set(torrentId, request); return request; })).then(function(logfiles) { if (logfiles.some(logfile => logfile.length <= 0)) throw 'one or both releases have invalid logfile'; if (logfiles.some(lf1 => logfiles.some(lf2 => lf1.length != lf2.length))) throw 'disc count mismatch'; const remarks = [ ]; for (let volumeNdx = 0; volumeNdx < logfiles[0].length; ++volumeNdx) { function addTrackRemark(trackNdx, remark) { if (!(trackNdx in volRemarks)) volRemarks[trackNdx] = [ ]; volRemarks[trackNdx].push(remark); } function processTrackValues(patterns, ...callbacks) { if (!Array.isArray(patterns) || patterns.length <= 0 || typeof callbacks[patterns.length] != 'function') return; const rxs = patterns.map(pattern => new RegExp('^[^\\S\\r\\n]+' + pattern + '\\s*$', 'm')); const values = trackRecords.map(trackRecords => trackRecords != null ? trackRecords.map(function(trackRecord, trackNdx) { trackRecord = rxs.map(rx => rx.exec(trackRecord)); for (let index = 0; index < trackRecord.length; ++index) if (trackRecord[index] != null) return typeof callbacks[index] == 'function' ? callbacks[index](trackRecord[index]) : trackRecord[index]; }) : [ ]); for (let trackNdx = 0; trackNdx < Math.max(values[0].length, values[1].length); ++trackNdx) callbacks[patterns.length](values[0][trackNdx], values[1][trackNdx], trackNdx); } const isRangeRip = logfiles.map(logfiles => rxRR.test(logfiles[volumeNdx])), volRemarks = { }; if (isRangeRip.some(Boolean)) remarks.push(`disc ${volumeNdx + 1} having at least one release as range rip, skipping peaks comparison`); let tocEntries = logfiles.map(logfile => logfile[volumeNdx].match(new RegExp(tocParser, 'gm'))); if (tocEntries.some(toc => toc == null)) throw `disc ${volumeNdx + 1} ToC not found for at least one release`; if (tocEntries[0].length != tocEntries[1].length) throw `disc ${volumeNdx + 1} ToC lengths mismatch`; tocEntries = tocEntries.map(tocEntriesMapper); const trackRecords = logfiles.map(logfiles => logfiles[volumeNdx].match(/^(?:Track)\s+\d+[^\S\r\n]*$\r?\n(?:^(?:[^\S\r\n]+.*)?$\r?\n)*/gm)); if (trackRecords.some((trackRecords, ndx) => !isRangeRip[ndx] && trackRecords == null)) throw `disc ${volumeNdx + 1} no track records could be extracted for at least one rip`; else if (!isRangeRip.some(Boolean) && trackRecords[0].length != trackRecords[1].length) throw `disc ${volumeNdx + 1} track records count mismatch (${trackRecords[0].length} <> ${trackRecords[1].length})`; const htoaCount = tocEntries.filter(tocEntries => tocEntries[0].startSector > 150).length; if (htoaCount > 0) remarks.push(`disc ${volumeNdx + 1} ${htoaCount < tocEntries.length ? 'one rip' : 'both rips'} possibly containing leading hidden track (ToC starting at non-zero offset)`); // Compare TOCs const maxTocShift = 40, maxTocDrift = 40, maxPeakDelta = 0.01; const tocShifts = tocEntries[0].map((_, trackNdx) => tocEntries[1][trackNdx].endSector - tocEntries[0][trackNdx].endSector); const tocShiftOf = shifts => Math.max(...shifts.map(Math.abs)); let tocShift = tocShifts.length > 1 ? tocShiftOf(tocShifts.slice(0, -1)) : undefined; tocShift = tocShifts.find(trackShift => Math.abs(trackShift) == tocShift) || 0; const hasPostGap = [tocShift + 150, tocShift - 150].includes(tocShifts[tocShifts.length - 1]); // ?? tocShift = !hasPostGap ? tocShiftOf(tocShifts) : tocShifts.length > 1 ? tocShiftOf(tocShifts.slice(0, -1)) : 0; const tocDrift = !hasPostGap ? Math.max(...tocShifts) - Math.min(...tocShifts) : tocShifts.length > 1 ? Math.max(...tocShifts.slice(0, -1)) - Math.min(...tocShifts.slice(0, -1)) : 0; console.assert(tocDrift >= 0); const label = `ToC comparison for tid${torrentIds[0]} and tid${torrentIds[1]} disc ${volumeNdx + 1}`; console.group(label); console.table(tocEntries[0].map((_, trackNdx) => ({ ['track#']: trackNdx + 1, ['start' + torrentIds[0]]: tocEntries[0][trackNdx].startSector, ['end' + torrentIds[0]]: tocEntries[0][trackNdx].endSector, ['length' + torrentIds[0]]: tocEntries[0][trackNdx].endSector + 1 - tocEntries[0][trackNdx].startSector, ['start' + torrentIds[1]]: tocEntries[1][trackNdx].startSector, ['end' + torrentIds[1]]: tocEntries[1][trackNdx].endSector, ['length' + torrentIds[1]]: tocEntries[1][trackNdx].endSector + 1 - tocEntries[1][trackNdx].startSector, ['tocShift']: tocShifts[trackNdx], ['assessment']: Math.abs(tocShifts[trackNdx]) > maxTocShift ? 'failed' : 'passed', }))); console.info(`ToC shift = ${tocShift} (${tocShift > maxTocShift ? 'failed' : 'passed'})`); console.info(`ToC drift = ${Math.max(...tocShifts)} - ${Math.min(...tocShifts)} = ${Math.max(...tocShifts) - Math.min(...tocShifts)} (${tocDrift > maxTocDrift ? 'failed' : 'passed'})`); console.groupEnd(label); if (tocShift > maxTocShift) throw `disc ${volumeNdx + 1} ToC shift above ${maxTocShift} sectors`; else if (tocDrift > maxTocDrift) throw `disc ${volumeNdx + 1} ToC drift above ${maxTocDrift} sectors`; else if (tocDrift > 0) remarks.push(`Disc ${volumeNdx + 1} shifted ToCs by ${tocShift} sectors with ${tocDrift} sectors drift`); else if (tocShifts[0] != 0) remarks.push(`Disc ${volumeNdx + 1} shifted ToCs by ${tocShift} sectors`); if (hasPostGap) remarks.push(`Disc ${volumeNdx + 1} with post-gap`); for (let trackNdx = 0; trackNdx < tocEntries[0].length; ++trackNdx) { // just informational const mismatches = [ ]; if (tocEntries[0][trackNdx].startSector != tocEntries[1][trackNdx].startSector) mismatches.push('offsets'); if (tocEntries[0][trackNdx].endSector - tocEntries[0][trackNdx].startSector != tocEntries[1][trackNdx].endSector - tocEntries[1][trackNdx].startSector) mismatches.push('lengths'); if (mismatches.length > 0) addTrackRemark(trackNdx, mismatches.join(' and ') + ' mismatch'); } // Compare pre-gaps - just informational if (!isRangeRip.some(Boolean)) processTrackValues([ '(?:Pre-gap length)\\s+' + msfTime, '(?:Pre-gap length)\\s*:\\s*' + msfTime, ], msfToSector, msfToSector, function(preGap1, preGap2, trackNdx) { if ((preGap1 || 0) != (preGap2 || 0)) addTrackRemark(trackNdx, 'pre-gaps mismatch'); }); // Compare peaks if (!isRangeRip.every(Boolean)) processTrackValues([ '(?:Peak level)\\s+(\\d+(?:\\.\\d+)?)\\s*\\%', '(?:Peak)\\s*:\\s*(\\d+(?:\\.\\d+)?)', ], m => parseFloat(m[1]) / 100, m => parseFloat(m[1]), function(peak1, peak2, trackNdx) { if (peak1 == undefined && !isRangeRip[0] || peak2 == undefined && !isRangeRip[1]) throw `disc ${volumeNdx + 1} track ${trackNdx + 1} peak missing or invalid format`; else if (!isRangeRip.some(Boolean)) if (Math.abs(peak2 - peak1) > maxPeakDelta) throw `disc ${volumeNdx + 1} track ${trackNdx + 1} peak difference above ${maxPeakDelta}`; else if (Math.round(peak2 * 1000) != Math.round(peak1 * 1000)) addTrackRemark(trackNdx, 'peak levels mismatch'); }); // Compare checksums - just informational if (!isRangeRip.every(Boolean)) processTrackValues([ '(?:Copy CRC)\\s+([\\da-fA-F]{8})', '(?:CRC32 hash)\\s*:\\s*([\\da-fA-F]{8})', ], m => parseInt(m[1], 16), m => parseInt(m[1], 16), function(checksum1, checksum2, trackNdx) { if (checksum1 == undefined && !isRangeRip[0] || checksum2 == undefined && !isRangeRip[1]) addTrackRemark(trackNdx, 'checksum missing or invalid format'); else if (!isRangeRip.some(Boolean) && checksum1 != checksum2) addTrackRemark(trackNdx, 'checksums mismatch'); }); // Compare AR signatures - just informational if (!isRangeRip.every(Boolean)) for (let v = 2; v > 0; --v) processTrackValues([ '.+?\\[([\\da-fA-F]{8})\\].+\\(AR v' + v + '\\)', '(?:AccurateRip v' + v + ' signature)\\s*:\\s*([\\da-fA-F]{8})', ], m => parseInt(m[1], 16), m => parseInt(m[1], 16), function(hash1, hash2, trackNdx) { if (hash1 == undefined && !isRangeRip[0] || hash2 == undefined && !isRangeRip[1]) /*addTrackRemark(trackNdx, 'AR v' + v + ' hash missing')*/; else if (!isRangeRip.some(Boolean) && hash1 != hash2) addTrackRemark(trackNdx, 'AR v' + v + ' hashes mismatch'); }); for (let trackNdx in volRemarks) remarks.push(`Disc ${volumeNdx + 1} track ${parseInt(trackNdx) + 1}: ${volRemarks[trackNdx].join(', ')}`); const timeStamps = logfiles.map(logfiles => /^(EAC|XLD) (?:extraction logfile from) (.+)$/m.exec(logfiles[volumeNdx])); if (timeStamps.every(Boolean) && timeStamps.map(timeStamp => timeStamp[0]).every((timeStamp, ndx, arr) => timeStamp == arr[0])) remarks.push(`Disc ${volumeNdx + 1} originates in same ripping session`); } if (remarks.filter(remark => remark.endsWith('originates in same ripping session')).length == logfiles[0].length) return true; return remarks; }); } const torrents = Array.prototype.filter.call(document.body.querySelectorAll('table#torrent_details > tbody > tr.torrent_row'), tr => (tr = tr.querySelector('td > a')) != null && /\b(?:FLAC)\b.+\b(?:Lossless)\b.+\b(?:Log) \(\-?\d+\s*\%\)/.test(tr.textContent)); if (torrents.length < 2) return; for (let tr of torrents) { let torrentId = /^torrent(\d+)$/.exec(tr.id); if (torrentId == null || !((torrentId = parseInt(torrentId[1])) > 0)) continue; const div = document.createElement('DIV'); div.innerHTML = ''; div.style = 'float: right; margin-left: 5pt; margin-right: 5pt; padding: 0; visibility: visible; cursor: pointer;'; div.className = 'compare-release'; div.onclick = function(evt) { console.assert(evt.currentTarget instanceof HTMLElement); const setActive = (elem, active = true) => { elem.children[0].children[0].setAttribute('fill', active ? 'orange' : 'gray') }; if (selected instanceof HTMLElement) { if (selected == evt.currentTarget) { selected = null; setActive(evt.currentTarget, false); } else { const target = evt.currentTarget; setActive(target, true); testSimilarity(...getTorrentIds(selected.parentNode.parentNode, target.parentNode.parentNode)).then(function(remarks) { if (remarks === true) return alert('Identical rips (dupes)'); let message = 'Releases can be duplicates (ToC shift/drift + peaks are too similar)'; if (remarks.length > 0) message += '\n\n' + (maxRemarks > 0 && remarks.length > maxRemarks ? remarks.slice(0, maxRemarks - 1).join('\n') + '\n...' : remarks.join('\n')); alert(message); }).catch(reason => { alert('Releases not duplicates for the reason ' + reason) }).then(function() { [selected, target].forEach(elem => { setActive(elem, false) }); selected = null; }); } } else setActive(selected = evt.currentTarget, true); }; div.title = 'Compare with different CD for similarity'; const anchor = tr.querySelector('span.torrent_action_buttons'); if (anchor != null) anchor.insertAdjacentElement('afterend', div); } const container = document.body.querySelector('table#torrent_details > tbody > tr.colhead_dark > td:first-of-type'); if (container == null) throw 'Torrent table header not found'; const span = document.createElement('SPAN'); span.className = 'brackets'; span.textContent = 'Find CD dupes'; span.style = 'margin-left: 5pt; margin-right: 5pt; float: right; cursor: pointer; font-size: 8pt;'; span.onclick = function(evt) { const compareWorkers = [ ]; torrents.forEach(function(torrent1, ndx1) { torrents.forEach(function(torrent2, ndx2) { if (ndx2 > ndx1) compareWorkers.push(testSimilarity(...getTorrentIds(torrent1, torrent2)) .then(remarks => [torrent1, torrent2], reason => 'distinct')); }); }); if (compareWorkers.length > 0) Promise.all(compareWorkers).then(function(results) { if ((results = results.filter(Array.isArray)).length > 0) try { function getEditionTitle(elem) { while (elem != null && !elem.classList.contains('edition')) elem = elem.previousElementSibling; if (elem != null && (elem = elem.querySelector('td.edition_info > strong')) != null) return elem.textContent.trim(); } results.forEach(function(sameTorrents, groupNdx) { const randColor = () => 0xD0 + Math.floor(Math.random() * (0xF8 - 0xD0)); const color = ['#dff', '#ffd', '#fdd', '#dfd', '#ddf', '#fdf'][groupNdx] || `rgb(${randColor()}, ${randColor()}, ${randColor()})`; for (let elem of sameTorrents) if ((elem = elem.querySelector('div.compare-release')) != null) { elem.style.padding = '2px'; elem.style.border = '1px solid #808080'; elem.style.borderRadius = '3px'; elem.style.backgroundColor = color; } }); alert('Similar CDs detected in these editions:\n\n' + results.map(sameTorrents => getEditionTitle(sameTorrents[0]) + '\n' + getEditionTitle(sameTorrents[1])).join('\n\n')); } catch (e) { alert(e) } else alert('No similar CDs detected'); }); }; container.append(span);