From 697bed23ed989f6ecac5cd248ad15c8a18bca2a1 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Tue, 25 Oct 2022 04:33:08 +0200 Subject: [PATCH] Disable http disk cache and implempent in-memory image cache (#2498) * Disable http disk cache and implempent in-memory image cache * Add comment about removing URL scheme prefix Co-authored-by: PikachuEXE * Add early return to clean up the code * Rewrite cache expiry logic with fallbacks * Move this change behind a CLI argument --experiments-disable-disk-cache * Replace CLI flag with a GUI setting * Improve warning message styling * ! Fix incompatibility with latest settings code * Use CSS instead of sass for the experimental settings * Return the error as JSON instead of throwing it * Inline restart prompt label and option names and values * Mention crash risk and recommend backups in the warning Co-authored-by: PikachuEXE --- _scripts/dev-runner.js | 8 +- src/main/ImageCache.js | 73 +++++++++++ src/main/index.js | 122 +++++++++++++++++- .../experimental-settings.css | 6 + .../experimental-settings.js | 73 +++++++++++ .../experimental-settings.vue | 30 +++++ .../ft-toggle-switch/ft-toggle-switch.js | 4 + .../ft-toggle-switch/ft-toggle-switch.vue | 2 +- src/renderer/views/Settings/Settings.js | 4 +- src/renderer/views/Settings/Settings.vue | 2 + static/locales/en-US.yaml | 6 + 11 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 src/main/ImageCache.js create mode 100644 src/renderer/components/experimental-settings/experimental-settings.css create mode 100644 src/renderer/components/experimental-settings/experimental-settings.js create mode 100644 src/renderer/components/experimental-settings/experimental-settings.vue diff --git a/_scripts/dev-runner.js b/_scripts/dev-runner.js index bedcb7f2..24857459 100644 --- a/_scripts/dev-runner.js +++ b/_scripts/dev-runner.js @@ -53,10 +53,12 @@ async function restartElectron() { electronProcess = spawn(electron, [ path.join(__dirname, '../dist/main.js'), - // '--enable-logging', Enable to show logs from all electron processes + // '--enable-logging', // Enable to show logs from all electron processes remoteDebugging ? '--inspect=9222' : '', - remoteDebugging ? '--remote-debugging-port=9223' : '', - ]) + remoteDebugging ? '--remote-debugging-port=9223' : '' + ], + // { stdio: 'inherit' } // required for logs to actually appear in the stdout + ) electronProcess.on('exit', (code, _) => { if (code === relaunchExitCode) { diff --git a/src/main/ImageCache.js b/src/main/ImageCache.js new file mode 100644 index 00000000..4a635b7d --- /dev/null +++ b/src/main/ImageCache.js @@ -0,0 +1,73 @@ +// cleanup expired images once every 5 mins +const CLEANUP_INTERVAL = 300_000 + +// images expire after 2 hours if no expiry information is found in the http headers +const FALLBACK_MAX_AGE = 7200 + +export class ImageCache { + constructor() { + this._cache = new Map() + + setInterval(this._cleanup.bind(this), CLEANUP_INTERVAL) + } + + add(url, mimeType, data, expiry) { + this._cache.set(url, { mimeType, data, expiry }) + } + + has(url) { + return this._cache.has(url) + } + + get(url) { + const entry = this._cache.get(url) + + if (!entry) { + // this should never happen as the `has` method should be used to check for the existence first + throw new Error(`No image cache entry for ${url}`) + } + + return { + data: entry.data, + mimeType: entry.mimeType + } + } + + _cleanup() { + // seconds since 1970-01-01 00:00:00 + const now = Math.trunc(Date.now() / 1000) + + for (const [key, entry] of this._cache.entries()) { + if (entry.expiry <= now) { + this._cache.delete(key) + } + } + } +} + +/** + * Extracts the cache expiry timestamp of image from HTTP headers + * @param {Record} headers + * @returns a timestamp in seconds + */ +export function extractExpiryTimestamp(headers) { + const maxAgeRegex = /max-age=([0-9]+)/ + + const cacheControl = headers['cache-control'] + if (cacheControl && maxAgeRegex.test(cacheControl)) { + let maxAge = parseInt(cacheControl.match(maxAgeRegex)[1]) + + if (headers.age) { + maxAge -= parseInt(headers.age) + } + + // we don't need millisecond precision, so we can store it as seconds to use less memory + return Math.trunc(Date.now() / 1000) + maxAge + } else if (headers.expires) { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expires + + return Math.trunc(Date.parse(headers.expires) / 1000) + } else { + return Math.trunc(Date.now() / 1000) + FALLBACK_MAX_AGE + } +} diff --git a/src/main/index.js b/src/main/index.js index ef8ab6c0..d1bc8f04 100644 --- a/src/main/index.js +++ b/src/main/index.js @@ -1,12 +1,14 @@ import { app, BrowserWindow, dialog, Menu, ipcMain, - powerSaveBlocker, screen, session, shell, nativeTheme + powerSaveBlocker, screen, session, shell, nativeTheme, net, protocol } from 'electron' import path from 'path' import cp from 'child_process' import { IpcChannels, DBActions, SyncEvents } from '../constants' import baseHandlers from '../datastores/handlers/base' +import { extractExpiryTimestamp, ImageCache } from './ImageCache' +import { existsSync } from 'fs' if (process.argv.includes('--version')) { app.exit() @@ -49,6 +51,17 @@ function runApp() { app.commandLine.appendSwitch('enable-file-cookies') app.commandLine.appendSwitch('ignore-gpu-blacklist') + // command line switches need to be added before the app ready event first + // that means we can't use the normal settings system as that is asynchonous, + // doing it synchronously ensures that we add it before the event fires + const replaceHttpCache = existsSync(`${app.getPath('userData')}/experiment-replace-http-cache`) + if (replaceHttpCache) { + // the http cache causes excessive disk usage during video playback + // we've got a custom image cache to make up for disabling the http cache + // experimental as it increases RAM use in favour of reduced disk use + app.commandLine.appendSwitch('disable-http-cache') + } + // See: https://stackoverflow.com/questions/45570589/electron-protocol-handler-not-working-on-windows // remove so we can register each time as we run the app. app.removeAsDefaultProtocolClient('freetube') @@ -149,6 +162,113 @@ function runApp() { }) }) + if (replaceHttpCache) { + // in-memory image cache + + const imageCache = new ImageCache() + + protocol.registerBufferProtocol('imagecache', (request, callback) => { + // Remove `imagecache://` prefix + const url = decodeURIComponent(request.url.substring(13)) + if (imageCache.has(url)) { + const cached = imageCache.get(url) + + // eslint-disable-next-line node/no-callback-literal + callback({ + mimeType: cached.mimeType, + data: cached.data + }) + return + } + + const newRequest = net.request({ + method: request.method, + url + }) + + // Electron doesn't allow certain headers to be set: + // https://www.electronjs.org/docs/latest/api/client-request#requestsetheadername-value + // also blacklist Origin and Referrer as we don't want to let YouTube know about them + const blacklistedHeaders = ['content-length', 'host', 'trailer', 'te', 'upgrade', 'cookie2', 'keep-alive', 'transfer-encoding', 'origin', 'referrer'] + + for (const header of Object.keys(request.headers)) { + if (!blacklistedHeaders.includes(header.toLowerCase())) { + newRequest.setHeader(header, request.headers[header]) + } + } + + newRequest.on('response', (response) => { + const chunks = [] + response.on('data', (chunk) => { + chunks.push(chunk) + }) + + response.on('end', () => { + const data = Buffer.concat(chunks) + + const expiryTimestamp = extractExpiryTimestamp(response.headers) + const mimeType = response.headers['content-type'] + + imageCache.add(url, mimeType, data, expiryTimestamp) + + // eslint-disable-next-line node/no-callback-literal + callback({ + mimeType, + data: data + }) + }) + + response.on('error', (error) => { + console.error('image cache error', error) + + // error objects don't get serialised properly + // https://stackoverflow.com/a/53624454 + + const errorJson = JSON.stringify(error, (key, value) => { + if (value instanceof Error) { + return { + // Pull all enumerable properties, supporting properties on custom Errors + ...value, + // Explicitly pull Error's non-enumerable properties + name: value.name, + message: value.message, + stack: value.stack + } + } + + return value + }) + + // eslint-disable-next-line node/no-callback-literal + callback({ + statusCode: response.statusCode ?? 400, + mimeType: 'application/json', + data: Buffer.from(errorJson) + }) + }) + }) + + newRequest.end() + }) + + const imageRequestFilter = { urls: ['https://*/*', 'http://*/*'] } + session.defaultSession.webRequest.onBeforeRequest(imageRequestFilter, (details, callback) => { + // the requests made by the imagecache:// handler to fetch the image, + // are allowed through, as their resourceType is 'other' + if (details.resourceType === 'image') { + // eslint-disable-next-line node/no-callback-literal + callback({ + redirectURL: `imagecache://${encodeURIComponent(details.url)}` + }) + } else { + // eslint-disable-next-line node/no-callback-literal + callback({}) + } + }) + + // --- end of `if experimentsDisableDiskCache` --- + } + await createWindow() if (isDev) { diff --git a/src/renderer/components/experimental-settings/experimental-settings.css b/src/renderer/components/experimental-settings/experimental-settings.css new file mode 100644 index 00000000..58e50c3a --- /dev/null +++ b/src/renderer/components/experimental-settings/experimental-settings.css @@ -0,0 +1,6 @@ +.experimental-warning { + text-align: center; + font-weight: bold; + padding-left: 4%; + padding-right: 4% +} diff --git a/src/renderer/components/experimental-settings/experimental-settings.js b/src/renderer/components/experimental-settings/experimental-settings.js new file mode 100644 index 00000000..83cbf708 --- /dev/null +++ b/src/renderer/components/experimental-settings/experimental-settings.js @@ -0,0 +1,73 @@ +import { closeSync, existsSync, openSync, rmSync } from 'fs' +import Vue from 'vue' +import { mapActions } from 'vuex' +import FtSettingsSection from '../ft-settings-section/ft-settings-section.vue' +import FtFlexBox from '../ft-flex-box/ft-flex-box.vue' +import FtToggleSwitch from '../ft-toggle-switch/ft-toggle-switch.vue' +import FtPrompt from '../ft-prompt/ft-prompt.vue' + +export default Vue.extend({ + name: 'ExperimentalSettings', + components: { + 'ft-settings-section': FtSettingsSection, + 'ft-flex-box': FtFlexBox, + 'ft-toggle-switch': FtToggleSwitch, + 'ft-prompt': FtPrompt + }, + data: function () { + return { + replaceHttpCacheLoading: true, + replaceHttpCache: false, + replaceHttpCachePath: '', + showRestartPrompt: false + } + }, + mounted: function () { + this.getUserDataPath().then((userData) => { + this.replaceHttpCachePath = `${userData}/experiment-replace-http-cache` + + this.replaceHttpCache = existsSync(this.replaceHttpCachePath) + this.replaceHttpCacheLoading = false + }) + }, + methods: { + updateReplaceHttpCache: function () { + this.replaceHttpCache = !this.replaceHttpCache + + if (this.replaceHttpCache) { + // create an empty file + closeSync(openSync(this.replaceHttpCachePath, 'w')) + } else { + rmSync(this.replaceHttpCachePath) + } + }, + + handleRestartPrompt: function (value) { + this.replaceHttpCache = value + this.showRestartPrompt = true + }, + + handleReplaceHttpCache: function (value) { + this.showRestartPrompt = false + + if (value === null || value === 'no') { + this.replaceHttpCache = !this.replaceHttpCache + return + } + + if (this.replaceHttpCache) { + // create an empty file + closeSync(openSync(this.replaceHttpCachePath, 'w')) + } else { + rmSync(this.replaceHttpCachePath) + } + + const { ipcRenderer } = require('electron') + ipcRenderer.send('relaunchRequest') + }, + + ...mapActions([ + 'getUserDataPath' + ]) + } +}) diff --git a/src/renderer/components/experimental-settings/experimental-settings.vue b/src/renderer/components/experimental-settings/experimental-settings.vue new file mode 100644 index 00000000..0e70671b --- /dev/null +++ b/src/renderer/components/experimental-settings/experimental-settings.vue @@ -0,0 +1,30 @@ + + +