'use strict'; const fs = require('fs'); const path = require('path'); const minimist = require('minimist'); const low = require('lowdb'); const Queue = require('better-queue'); const FileSync = require('lowdb/adapters/FileSync'); const colors = require('colors'); const chokidar = require('chokidar'); const hbjs = require('handbrake-js'); const MINUTE = 60 * 1000; const HOUR = 60 * MINUTE; const DAY = 24 * HOUR; const WEEK = 7 * DAY; const CLEANER = HOUR; colors.setTheme({ silly: 'rainbow', input: 'grey', verbose: 'cyan', prompt: 'grey', info: 'grey', data: 'grey', help: 'cyan', warn: 'yellow', success: 'green', failure: 'red', debug: 'blue', delete: 'red', changed: 'cyan', error: 'red', }); const settings = require('./data/settings.json'); const args = minimist(process.argv.slice(2), { alias: { h: 'help', v: 'version', c: 'config', i: 'ignoreInitial', t: 'test', }, }); const version = () => { const pkg = require('./package.json'); return console.log(colors.help(pkg.version)); }; const help = () => { let message = `Usage: \tnpm run movies \tnpm run tvshows \tnpm run pr0n \tnpm run all \tnpm run help \tnpm run version\n`; return console.log(colors.help(message)); }; String.prototype.toTitleCase = function () { var i, j, str, lowers, uppers; str = this.replace(/([^\W_]+[^\s-]*) */g, function (txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); }); // Certain minor words should be left lowercase unless // they are the first or last words in the string lowers = (settings && settings.titlecase && settings.titlecase.lowers) || []; for (i = 0, j = lowers.length; i < j; i++) str = str.replace(new RegExp('\\s' + lowers[i] + '\\s', 'g'), function (txt) { return txt.toLowerCase(); }); // Certain words such as initialisms or acronyms should be left uppercase uppers = (settings && settings.titlecase && settings.titlecase.uppers) || []; for (i = 0, j = uppers.length; i < j; i++) str = str.replace(new RegExp('\\b' + uppers[i] + '\\b', 'g'), uppers[i].toUpperCase()); return str; }; // get the db from the path const getDbForDir = (dir, opts) => { let options = opts || Object.assign({}, defaults, paths[dir]); // baseline the options let adapter = new FileSync(options.database); // init the db adapter let db = low(adapter); // connect the db return db; }; // find a file in the db const findFile = (db, file) => { return db .get('files') .find({ input: file, }) .value(); // does it already exist? }; // set a file in the db const setFile = (db, file, payload) => { if (!payload && typeof file === 'object') return db.get('files').push(file).write(); return db .get('files') .find({ input: file, }) .assign(payload) .write(); }; // remove a file from the db const removeFile = (db, file, soft = true) => { if (soft) return db .get('files') .find({ input: file, }) .assign({ status: 'deleted', date: new Date() }) .write(); // remove file from database return db .get('files') .remove({ input: file, }) .write(); // remove file from database }; // write process output const processOutput = (str) => { process.stdout.clearLine(); process.stdout.cursorTo(0); process.stdout.write(str); }; const getFilesize = async (file) => { let stats = fs.existsSync(file) ? fs.statSync(file) : false; let bytes = stats ? stats['size'] : 0; return bytes && bytes > 0 ? bytes / 1024 : 0; }; // spawn a handbrake const processWithHandbrake = (input, output, preset) => { return new Promise((resolve, reject) => { const inputName = path.basename(input); const outputName = path.basename(output); hbjs .spawn({ input: input, output: output, preset: preset, }) .on('start', (err) => { processOutput(colors.info(`-> Starting "${outputName}" (${new Date()} with "${preset}")`)); }) .on('error', (err) => { processOutput(colors.error(`-> Errored "${outputName}" (${new Date()}: ${err.message || err})\n`)); reject(err); }) .on('progress', (progress) => { processOutput(colors.warn(`-> Transcoding "${outputName}" (${progress.percentComplete}%, ETA: ${progress.eta})`)); }) .on('cancelled', () => { processOutput(colors.failure(`-> Cancelled "${outputName}" (${new Date()})\n`)); reject(new Error(`Processing "${outputName}" cancelled`)); }) .on('complete', () => { processOutput(colors.success(`-> Completed "${outputName}" (${new Date()} with "${preset}")\n`)); resolve(true); }); }); }; const processFile = async (file) => { let db, options, preset = 'Fast 1080p30', clean = { '.': ' ', }, ext = 'm4v', filename = path.parse(file).name, input = file, output = filename, titlecase = false, folder = false, destination = ''; // determine the presets and clean object for (let i = 0, l = dirs.length; i < l; i++) { let dir = dirs[i]; // pointer to the dir if (file && dir && file.indexOf(dir) > -1) { // is this in this path? options = Object.assign({}, defaults, paths[dir]); // baseline the options db = getDbForDir(dir, options); // init the db connection let found = findFile(db, file); // does it already exist? if (found && ((found.status && found.status === 'success') || found.processed)) { // was it already processed? return false; // break this loop } else if (!found) { // was it found? .. nope processOutput(colors.warn(`-> Processing "${path.basename(file)}" (${new Date()})`)); setFile(db, { input: file, output: '', status: '', date: new Date(), }); // push onto the list an entry } else { processOutput(colors.warn(`-> Re-processing "${path.basename(file)}" (${new Date()})`)); setFile(db, file, { status: '', date: new Date(), }); // set the status to blank } preset = options.preset; // apply the specifics ... clean = options.clean; // ... ext = options.ext; // ... titlecase = !!options.titlecase; // ... folder = !!options.folder; // ... destination = options.destination; // ... break; // break the loop } } // clean the output name let cleanKeys = Object.keys(clean); for (let c in cleanKeys) { let key = cleanKeys[c]; let val = clean[key]; let re = new RegExp(key, 'gi'); output = output.replace(re, val); } // baseline the output name output = output.trim(); // trim the whitespace output = output.replace(/(\d{4})$/, `($1)`); // if there is a date at the end wrap it // do we have a sub folder option? if (folder) { let match = output.match(/^(.*?)?\./gm); // get the name for the file before the first . folder = match && match.length > 0 ? match[0].slice(0, -1) : false; // get just the stuff before the dot ... or false } // do we have title case enabled? if (options.titlecase) output = output.toTitleCase(); // titlecase that string // baseline the target let target = destination + (folder ? '/' + folder : ''); // setup the target location output = target + '/' + output + '.' + ext; // update the new name // do we already have an existing output that matches? if (fs.existsSync(output)) { let outputSize = await getFilesize(output); //get output filesize let inputSize = await getFilesize(input); //get output filesize if (outputSize > inputSize / 4) { //make sure its bigger than 100k processOutput(colors.warn(`-> Skipping "${path.basename(file)}" (already processed)\n`)); setFile(db, file, { output: output, processed: true, status: 'success', date: new Date(), }); // update database with status return false; } } //process.stdout.write('\n'); // send a new line // update database with output name setFile(db, file, { output: output, }); // create parent if required if (target && !fs.existsSync(target)) { processOutput(colors.info(`-> Creating parent directory "${path.basename(file)}" ("${target}")`)); fs.mkdirSync(target, { recursive: true, }); } // spawn handbrake if (!test) { try { await processWithHandbrake(input, output, preset); setFile(db, file, { status: 'success', processed: true, date: new Date(), }); // update database with status } catch (err) { setFile(db, file, { status: 'failure', processed: false, date: new Date(), }); // update database with status } } return true; // when complete return true }; // cleanup removes from the db const cleanup = (file) => { let db; for (let i = 0, l = dirs.length; i < l; i++) { let dir = dirs[i]; // pointer to the dir if (file && dir && file.indexOf(dir) > -1) { db = getDbForDir(dir); // init the db connection let exists = fs.existsSync(file); // check if the file actually exists if (!exists) removeFile(db, file, true); //soft remove } } }; // handle args if (args.version) return version(); // show version if (args.help || !args.config) return help(); // show help const ignoreInitial = args.hasOwnProperty('ignoreInitial') ? args.ignoreInitial : false; // ignore initial files const test = args.hasOwnProperty('test') ? args.test : false; // do a dry run ... just log // get our paths const paths = require(args.config); // init our defaults const defaults = { preset: 'Fast 1080p30', clean: {}, titlecase: false, folder: false, database: 'data/db.json', }; // setup watcher options const opts = { ignored: /(^|[\/\\])\..|([s|S]ample\.*)/, ignoreInitial: ignoreInitial, persistent: true, usePolling: true, interval: 10 * 1000, depth: 1, awaitWriteFinish: { stabilityThreshold: 3000, pollInterval: 1000, }, ignorePermissionErrors: false, atomic: true, }; // parse the paths to dirs const dirs = Object.keys(paths); // initialize watches and db then start the watcher const main = async () => { var queue = new Queue( async (input, cb) => { // init the queue let result = await processFile(input); // process the queue cb(null, result); }, { batchSize: 1, concurrent: 1, maxRetries: 3, retryDelay: 5000, } ); var watches = []; // array of things to watch for (let d in dirs) { // loop the dirs let dir = dirs[d]; // pointer let options = Object.assign({}, defaults, paths[dir]); // baseline the options let db = getDbForDir(dir); db.defaults({ files: [], }).write(); // init the database for (let e in options.exts) { // loop the exts to watch let ext = options.exts[e]; // alias the ext watches.push(`${dir}/**/*.${ext}`); // push the watch } } const watcher = chokidar.watch(watches, opts); // init our watcher console.log(colors.info('Watching:', watches)); watcher .on('add', async (file) => { // when a file is added ... queue.push(file); // push the file onto the queue to be processed }) .on('change', (file) => { // when a file changes ... console.log(colors.changed(`-> Changed "${file}" (${new Date()})`)); }) .on('unlink', async (file) => { // when a file is removed ... console.log(colors.delete(`-> Removed "${file}" (${new Date()})`)); cleanup(file); }) .on('error', (error) => { console.error(colors.error(`-> Errored "${file}"(${new Date()}: ${error.message || error})`)); }); }; const purge = () => { let ago = new Date(Date.now() - DAY); let db; console.log(colors.info(`Checking for and "deleted" records older then ${ago}`)); for (let i = 0, l = dirs.length; i < l; i++) { let dir = dirs[i]; // pointer to the dir db = getDbForDir(dir); // init the db connection let files = db .get('files') .filter((file) => file.status && file.status === 'deleted' && file.processed && file.date && new Date(file.date).getTime() < ago.getTime()); for (let file of files) { console.log(colors.delete(`-> Purging "${file.input}" (${new Date()})`)); if (file && file.input) removeFile(db, file.input, false); } } setTimeout(() => { purge(); }, CLEANER); }; const prune = () => { let db; console.log(colors.info('Checking for any "processed" files that need pruning.')); for (let i = 0, l = dirs.length; i < l; i++) { let dir = dirs[i]; // pointer to the dir db = getDbForDir(dir); // init the db connection let files = db.get('files').filter((file) => file && file.processed); for (let file of files) { let exists = fs.existsSync(file.input); // check if the file actually exists if (!exists) { console.log(colors.delete(`-> Pruning "${file.input}" (${new Date()})`)); removeFile(db, file.input, false); //soft remove } } } }; (async () => { main(); purge(); prune(); })().catch((err) => { console.log(colors.error(`Error: ${err.message || err}`)); });