Disable http disk cache and implempent in-memory image cache (#2498)

* Disable http disk cache and implempent in-memory image cache

* Add comment about removing URL scheme prefix

Co-authored-by: PikachuEXE <pikachuexe@gmail.com>

* Add early return to clean up the code

* Rewrite cache expiry logic with fallbacks

* Move this change behind a CLI argument --experiments-disable-disk-cache

* Replace CLI flag with a GUI setting

* Improve warning message styling

* ! Fix incompatibility with latest settings code

* Use CSS instead of sass for the experimental settings

* Return the error as JSON instead of throwing it

* Inline restart prompt label and option names and values

* Mention crash risk and recommend backups in the warning

Co-authored-by: PikachuEXE <pikachuexe@gmail.com>
This commit is contained in:
absidue 2022-10-25 04:33:08 +02:00 committed by GitHub
parent 9ed512776c
commit 697bed23ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 324 additions and 6 deletions

View File

@ -53,10 +53,12 @@ async function restartElectron() {
electronProcess = spawn(electron, [ electronProcess = spawn(electron, [
path.join(__dirname, '../dist/main.js'), path.join(__dirname, '../dist/main.js'),
// '--enable-logging', Enable to show logs from all electron processes // '--enable-logging', // Enable to show logs from all electron processes
remoteDebugging ? '--inspect=9222' : '', remoteDebugging ? '--inspect=9222' : '',
remoteDebugging ? '--remote-debugging-port=9223' : '', remoteDebugging ? '--remote-debugging-port=9223' : ''
]) ],
// { stdio: 'inherit' } // required for logs to actually appear in the stdout
)
electronProcess.on('exit', (code, _) => { electronProcess.on('exit', (code, _) => {
if (code === relaunchExitCode) { if (code === relaunchExitCode) {

73
src/main/ImageCache.js Normal file
View File

@ -0,0 +1,73 @@
// cleanup expired images once every 5 mins
const CLEANUP_INTERVAL = 300_000
// images expire after 2 hours if no expiry information is found in the http headers
const FALLBACK_MAX_AGE = 7200
export class ImageCache {
constructor() {
this._cache = new Map()
setInterval(this._cleanup.bind(this), CLEANUP_INTERVAL)
}
add(url, mimeType, data, expiry) {
this._cache.set(url, { mimeType, data, expiry })
}
has(url) {
return this._cache.has(url)
}
get(url) {
const entry = this._cache.get(url)
if (!entry) {
// this should never happen as the `has` method should be used to check for the existence first
throw new Error(`No image cache entry for ${url}`)
}
return {
data: entry.data,
mimeType: entry.mimeType
}
}
_cleanup() {
// seconds since 1970-01-01 00:00:00
const now = Math.trunc(Date.now() / 1000)
for (const [key, entry] of this._cache.entries()) {
if (entry.expiry <= now) {
this._cache.delete(key)
}
}
}
}
/**
* Extracts the cache expiry timestamp of image from HTTP headers
* @param {Record<string, string>} headers
* @returns a timestamp in seconds
*/
export function extractExpiryTimestamp(headers) {
const maxAgeRegex = /max-age=([0-9]+)/
const cacheControl = headers['cache-control']
if (cacheControl && maxAgeRegex.test(cacheControl)) {
let maxAge = parseInt(cacheControl.match(maxAgeRegex)[1])
if (headers.age) {
maxAge -= parseInt(headers.age)
}
// we don't need millisecond precision, so we can store it as seconds to use less memory
return Math.trunc(Date.now() / 1000) + maxAge
} else if (headers.expires) {
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expires
return Math.trunc(Date.parse(headers.expires) / 1000)
} else {
return Math.trunc(Date.now() / 1000) + FALLBACK_MAX_AGE
}
}

View File

@ -1,12 +1,14 @@
import { import {
app, BrowserWindow, dialog, Menu, ipcMain, app, BrowserWindow, dialog, Menu, ipcMain,
powerSaveBlocker, screen, session, shell, nativeTheme powerSaveBlocker, screen, session, shell, nativeTheme, net, protocol
} from 'electron' } from 'electron'
import path from 'path' import path from 'path'
import cp from 'child_process' import cp from 'child_process'
import { IpcChannels, DBActions, SyncEvents } from '../constants' import { IpcChannels, DBActions, SyncEvents } from '../constants'
import baseHandlers from '../datastores/handlers/base' import baseHandlers from '../datastores/handlers/base'
import { extractExpiryTimestamp, ImageCache } from './ImageCache'
import { existsSync } from 'fs'
if (process.argv.includes('--version')) { if (process.argv.includes('--version')) {
app.exit() app.exit()
@ -49,6 +51,17 @@ function runApp() {
app.commandLine.appendSwitch('enable-file-cookies') app.commandLine.appendSwitch('enable-file-cookies')
app.commandLine.appendSwitch('ignore-gpu-blacklist') app.commandLine.appendSwitch('ignore-gpu-blacklist')
// command line switches need to be added before the app ready event first
// that means we can't use the normal settings system as that is asynchonous,
// doing it synchronously ensures that we add it before the event fires
const replaceHttpCache = existsSync(`${app.getPath('userData')}/experiment-replace-http-cache`)
if (replaceHttpCache) {
// the http cache causes excessive disk usage during video playback
// we've got a custom image cache to make up for disabling the http cache
// experimental as it increases RAM use in favour of reduced disk use
app.commandLine.appendSwitch('disable-http-cache')
}
// See: https://stackoverflow.com/questions/45570589/electron-protocol-handler-not-working-on-windows // See: https://stackoverflow.com/questions/45570589/electron-protocol-handler-not-working-on-windows
// remove so we can register each time as we run the app. // remove so we can register each time as we run the app.
app.removeAsDefaultProtocolClient('freetube') app.removeAsDefaultProtocolClient('freetube')
@ -149,6 +162,113 @@ function runApp() {
}) })
}) })
if (replaceHttpCache) {
// in-memory image cache
const imageCache = new ImageCache()
protocol.registerBufferProtocol('imagecache', (request, callback) => {
// Remove `imagecache://` prefix
const url = decodeURIComponent(request.url.substring(13))
if (imageCache.has(url)) {
const cached = imageCache.get(url)
// eslint-disable-next-line node/no-callback-literal
callback({
mimeType: cached.mimeType,
data: cached.data
})
return
}
const newRequest = net.request({
method: request.method,
url
})
// Electron doesn't allow certain headers to be set:
// https://www.electronjs.org/docs/latest/api/client-request#requestsetheadername-value
// also blacklist Origin and Referrer as we don't want to let YouTube know about them
const blacklistedHeaders = ['content-length', 'host', 'trailer', 'te', 'upgrade', 'cookie2', 'keep-alive', 'transfer-encoding', 'origin', 'referrer']
for (const header of Object.keys(request.headers)) {
if (!blacklistedHeaders.includes(header.toLowerCase())) {
newRequest.setHeader(header, request.headers[header])
}
}
newRequest.on('response', (response) => {
const chunks = []
response.on('data', (chunk) => {
chunks.push(chunk)
})
response.on('end', () => {
const data = Buffer.concat(chunks)
const expiryTimestamp = extractExpiryTimestamp(response.headers)
const mimeType = response.headers['content-type']
imageCache.add(url, mimeType, data, expiryTimestamp)
// eslint-disable-next-line node/no-callback-literal
callback({
mimeType,
data: data
})
})
response.on('error', (error) => {
console.error('image cache error', error)
// error objects don't get serialised properly
// https://stackoverflow.com/a/53624454
const errorJson = JSON.stringify(error, (key, value) => {
if (value instanceof Error) {
return {
// Pull all enumerable properties, supporting properties on custom Errors
...value,
// Explicitly pull Error's non-enumerable properties
name: value.name,
message: value.message,
stack: value.stack
}
}
return value
})
// eslint-disable-next-line node/no-callback-literal
callback({
statusCode: response.statusCode ?? 400,
mimeType: 'application/json',
data: Buffer.from(errorJson)
})
})
})
newRequest.end()
})
const imageRequestFilter = { urls: ['https://*/*', 'http://*/*'] }
session.defaultSession.webRequest.onBeforeRequest(imageRequestFilter, (details, callback) => {
// the requests made by the imagecache:// handler to fetch the image,
// are allowed through, as their resourceType is 'other'
if (details.resourceType === 'image') {
// eslint-disable-next-line node/no-callback-literal
callback({
redirectURL: `imagecache://${encodeURIComponent(details.url)}`
})
} else {
// eslint-disable-next-line node/no-callback-literal
callback({})
}
})
// --- end of `if experimentsDisableDiskCache` ---
}
await createWindow() await createWindow()
if (isDev) { if (isDev) {

View File

@ -0,0 +1,6 @@
.experimental-warning {
text-align: center;
font-weight: bold;
padding-left: 4%;
padding-right: 4%
}

View File

@ -0,0 +1,73 @@
import { closeSync, existsSync, openSync, rmSync } from 'fs'
import Vue from 'vue'
import { mapActions } from 'vuex'
import FtSettingsSection from '../ft-settings-section/ft-settings-section.vue'
import FtFlexBox from '../ft-flex-box/ft-flex-box.vue'
import FtToggleSwitch from '../ft-toggle-switch/ft-toggle-switch.vue'
import FtPrompt from '../ft-prompt/ft-prompt.vue'
export default Vue.extend({
name: 'ExperimentalSettings',
components: {
'ft-settings-section': FtSettingsSection,
'ft-flex-box': FtFlexBox,
'ft-toggle-switch': FtToggleSwitch,
'ft-prompt': FtPrompt
},
data: function () {
return {
replaceHttpCacheLoading: true,
replaceHttpCache: false,
replaceHttpCachePath: '',
showRestartPrompt: false
}
},
mounted: function () {
this.getUserDataPath().then((userData) => {
this.replaceHttpCachePath = `${userData}/experiment-replace-http-cache`
this.replaceHttpCache = existsSync(this.replaceHttpCachePath)
this.replaceHttpCacheLoading = false
})
},
methods: {
updateReplaceHttpCache: function () {
this.replaceHttpCache = !this.replaceHttpCache
if (this.replaceHttpCache) {
// create an empty file
closeSync(openSync(this.replaceHttpCachePath, 'w'))
} else {
rmSync(this.replaceHttpCachePath)
}
},
handleRestartPrompt: function (value) {
this.replaceHttpCache = value
this.showRestartPrompt = true
},
handleReplaceHttpCache: function (value) {
this.showRestartPrompt = false
if (value === null || value === 'no') {
this.replaceHttpCache = !this.replaceHttpCache
return
}
if (this.replaceHttpCache) {
// create an empty file
closeSync(openSync(this.replaceHttpCachePath, 'w'))
} else {
rmSync(this.replaceHttpCachePath)
}
const { ipcRenderer } = require('electron')
ipcRenderer.send('relaunchRequest')
},
...mapActions([
'getUserDataPath'
])
}
})

View File

@ -0,0 +1,30 @@
<template>
<ft-settings-section
:title="$t('Settings.Experimental Settings.Experimental Settings')"
>
<p class="experimental-warning">
{{ $t('Settings.Experimental Settings.Warning') }}
</p>
<ft-flex-box>
<ft-toggle-switch
tooltip-position="top"
:label="$t('Settings.Experimental Settings.Replace HTTP Cache')"
:compact="true"
:default-value="replaceHttpCache"
:disabled="replaceHttpCacheLoading"
:tooltip="$t('Tooltips.Experimental Settings.Replace HTTP Cache')"
@change="handleRestartPrompt"
/>
</ft-flex-box>
<ft-prompt
v-if="showRestartPrompt"
:label="$t('Settings[\'The app needs to restart for changes to take effect. Restart and apply change?\']')"
:option-names="[$t('Yes'), $t('No')]"
:option-values="['yes', 'no']"
@click="handleReplaceHttpCache"
/>
</ft-settings-section>
</template>
<script src="./experimental-settings.js" />
<style scoped src="./experimental-settings.css" />

View File

@ -26,6 +26,10 @@ export default Vue.extend({
tooltip: { tooltip: {
type: String, type: String,
default: '' default: ''
},
tooltipPosition: {
type: String,
default: 'bottom-left'
} }
}, },
data: function () { data: function () {

View File

@ -27,7 +27,7 @@
<ft-tooltip <ft-tooltip
v-if="tooltip !== ''" v-if="tooltip !== ''"
class="selectTooltip" class="selectTooltip"
position="bottom-left" :position="tooltipPosition"
:tooltip="tooltip" :tooltip="tooltip"
/> />
</label> </label>

View File

@ -13,6 +13,7 @@ import DistractionSettings from '../../components/distraction-settings/distracti
import ProxySettings from '../../components/proxy-settings/proxy-settings.vue' import ProxySettings from '../../components/proxy-settings/proxy-settings.vue'
import SponsorBlockSettings from '../../components/sponsor-block-settings/sponsor-block-settings.vue' import SponsorBlockSettings from '../../components/sponsor-block-settings/sponsor-block-settings.vue'
import ParentControlSettings from '../../components/parental-control-settings/parental-control-settings.vue' import ParentControlSettings from '../../components/parental-control-settings/parental-control-settings.vue'
import ExperimentalSettings from '../../components/experimental-settings/experimental-settings.vue'
export default Vue.extend({ export default Vue.extend({
name: 'Settings', name: 'Settings',
@ -30,7 +31,8 @@ export default Vue.extend({
'proxy-settings': ProxySettings, 'proxy-settings': ProxySettings,
'sponsor-block-settings': SponsorBlockSettings, 'sponsor-block-settings': SponsorBlockSettings,
'download-settings': DownloadSettings, 'download-settings': DownloadSettings,
'parental-control-settings': ParentControlSettings 'parental-control-settings': ParentControlSettings,
'experimental-settings': ExperimentalSettings
}, },
computed: { computed: {
usingElectron: function () { usingElectron: function () {

View File

@ -23,6 +23,8 @@
<parental-control-settings /> <parental-control-settings />
<hr> <hr>
<sponsor-block-settings /> <sponsor-block-settings />
<hr v-if="usingElectron">
<experimental-settings v-if="usingElectron" />
</div> </div>
</template> </template>

View File

@ -406,6 +406,10 @@ Settings:
Download Behavior: Download Behavior Download Behavior: Download Behavior
Download in app: Download in app Download in app: Download in app
Open in web browser: Open in web browser Open in web browser: Open in web browser
Experimental Settings:
Experimental Settings: Experimental Settings
Warning: These settings are experimental, they make cause crashes while enabled. Making backups is highly recommended. Use at your own risk!
Replace HTTP Cache: Replace HTTP Cache
About: About:
#On About page #On About page
About: About About: About
@ -774,6 +778,8 @@ Tooltips:
Privacy Settings: Privacy Settings:
Remove Video Meta Files: When enabled, FreeTube automatically deletes meta files created during video playback, Remove Video Meta Files: When enabled, FreeTube automatically deletes meta files created during video playback,
when the watch page is closed. when the watch page is closed.
Experimental Settings:
Replace HTTP Cache: Disables Electron's disk based HTTP cache and enables a custom in-memory image cache. Will lead to increased RAM usage.
# Toast Messages # Toast Messages
Local API Error (Click to copy): Local API Error (Click to copy) Local API Error (Click to copy): Local API Error (Click to copy)