diff --git a/app/javascript/mastodon/components/status.jsx b/app/javascript/mastodon/components/status.jsx index e1728910e..4a5ad54a1 100644 --- a/app/javascript/mastodon/components/status.jsx +++ b/app/javascript/mastodon/components/status.jsx @@ -546,7 +546,7 @@ class Status extends ImmutablePureComponent { const visibilityIcon = visibilityIconInfo[status.get('visibility')]; const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status); - const expanded = !status.get('hidden') + const expanded = !status.get('hidden') || status.get('spoiler_text').length === 0; return ( diff --git a/app/javascript/mastodon/features/status/components/detailed_status.jsx b/app/javascript/mastodon/features/status/components/detailed_status.jsx index e41a042c7..3927aba2f 100644 --- a/app/javascript/mastodon/features/status/components/detailed_status.jsx +++ b/app/javascript/mastodon/features/status/components/detailed_status.jsx @@ -293,7 +293,7 @@ class DetailedStatus extends ImmutablePureComponent { } const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status); - const expanded = !status.get('hidden') + const expanded = !status.get('hidden') || status.get('spoiler_text').length === 0; return (
diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index b88cf662c..ed5425a3b 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -41,11 +41,14 @@ module Paperclip @output_options['vframes'] = 1 when 'mp4' unless eligible_to_passthrough?(metadata) - bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL) / 1_000 - - @output_options['b:v'] = "#{bitrate}k" - @output_options['maxrate'] = "#{bitrate + 192}k" - @output_options['bufsize'] = "#{bitrate * 5}k" + size_limit_in_bits = MediaAttachment::VIDEO_LIMIT * 8 + desired_bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL).floor + maximum_bitrate = (size_limit_in_bits / metadata.duration).floor - 192_000 # Leave some space for the audio stream + bitrate = [desired_bitrate, maximum_bitrate].min + + @output_options['b:v'] = bitrate + @output_options['maxrate'] = bitrate + 192_000 + @output_options['bufsize'] = bitrate * 5 if high_vfr?(metadata) @output_options['vsync'] = 'vfr' diff --git a/streaming/index.js b/streaming/index.js index 2f784f248..254eed38d 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -152,6 +152,28 @@ const redisConfigFromEnv = (env) => { }; }; +const PUBLIC_CHANNELS = [ + 'public', + 'public:media', + 'public:local', + 'public:local:media', + 'public:remote', + 'public:remote:media', + 'hashtag', + 'hashtag:local', +]; + +// Used for priming the counters/gauges for the various metrics that are +// per-channel +const CHANNEL_NAMES = [ + 'system', + 'user', + 'user:notification', + 'list', + 'direct', + ...PUBLIC_CHANNELS +]; + const startServer = async () => { const app = express(); @@ -203,9 +225,6 @@ const startServer = async () => { labelNames: ['type'], }); - connectedClients.set({ type: 'websocket' }, 0); - connectedClients.set({ type: 'eventsource' }, 0); - const connectedChannels = new metrics.Gauge({ name: 'connected_channels', help: 'The number of channels the streaming server is streaming to', @@ -217,6 +236,35 @@ const startServer = async () => { help: 'The number of Redis channels the streaming server is subscribed to', }); + const redisMessagesReceived = new metrics.Counter({ + name: 'redis_messages_received_total', + help: 'The total number of messages the streaming server has received from redis subscriptions' + }); + + const messagesSent = new metrics.Counter({ + name: 'messages_sent_total', + help: 'The total number of messages the streaming server sent to clients per connection type', + labelNames: [ 'type' ] + }); + + // Prime the gauges so we don't loose metrics between restarts: + redisSubscriptions.set(0); + connectedClients.set({ type: 'websocket' }, 0); + connectedClients.set({ type: 'eventsource' }, 0); + + // For each channel, initialize the gauges at zero; There's only a finite set of channels available + CHANNEL_NAMES.forEach(( channel ) => { + connectedChannels.set({ type: 'websocket', channel }, 0); + connectedChannels.set({ type: 'eventsource', channel }, 0); + }) + + // Prime the counters so that we don't loose metrics between restarts. + // Unfortunately counters don't support the set() API, so instead I'm using + // inc(0) to achieve the same result. + redisMessagesReceived.inc(0); + messagesSent.inc({ type: 'websocket' }, 0); + messagesSent.inc({ type: 'eventsource' }, 0); + // When checking metrics in the browser, the favicon is requested this // prevents the request from falling through to the API Router, which would // error for this endpoint: @@ -262,6 +310,8 @@ const startServer = async () => { * @param {string} message */ const onRedisMessage = (channel, message) => { + redisMessagesReceived.inc(); + const callbacks = subs[channel]; log.silly(`New message on channel ${redisPrefix}${channel}`); @@ -490,17 +540,6 @@ const startServer = async () => { } }; - const PUBLIC_CHANNELS = [ - 'public', - 'public:media', - 'public:local', - 'public:local:media', - 'public:remote', - 'public:remote:media', - 'hashtag', - 'hashtag:local', - ]; - /** * @param {any} req * @param {string|undefined} channelName @@ -705,11 +744,12 @@ const startServer = async () => { * @param {any} req * @param {function(string, string): void} output * @param {undefined | function(string[], SubscriptionListener): void} attachCloseHandler + * @param {'websocket' | 'eventsource'} destinationType * @param {boolean=} needsFiltering * @param {boolean=} allowLocalOnly * @returns {SubscriptionListener} */ - const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, allowLocalOnly = false) => { + const streamFrom = (ids, req, output, attachCloseHandler, destinationType, needsFiltering = false, allowLocalOnly = false) => { const accountId = req.accountId || req.remoteAddress; log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`); @@ -718,6 +758,8 @@ const startServer = async () => { // TODO: Replace "string"-based delete payloads with object payloads: const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload; + messagesSent.labels({ type: destinationType }).inc(1); + log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload}`); output(event, encodedPayload); }; @@ -1038,7 +1080,7 @@ const startServer = async () => { const onSend = streamToHttp(req, res); const onEnd = streamHttpEnd(req, subscriptionHeartbeat(channelIds)); - streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering, options.allowLocalOnly); + streamFrom(channelIds, req, onSend, onEnd, 'eventsource', options.needsFiltering, options.allowLocalOnly); }).catch(err => { log.verbose(req.requestId, 'Subscription error:', err.toString()); httpNotFound(res); @@ -1262,7 +1304,7 @@ const startServer = async () => { const onSend = streamToWs(request, socket, streamNameFromChannelName(channelName, params)); const stopHeartbeat = subscriptionHeartbeat(channelIds); - const listener = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering, options.allowLocalOnly); + const listener = streamFrom(channelIds, request, onSend, undefined, 'websocket', options.needsFiltering, options.allowLocalOnly); connectedChannels.labels({ type: 'websocket', channel: channelName }).inc(); diff --git a/yarn.lock b/yarn.lock index ba735f86f..bbe1580df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1298,7 +1298,12 @@ dependencies: eslint-visitor-keys "^3.3.0" -"@eslint-community/regexpp@^4.5.1", "@eslint-community/regexpp@^4.6.1": +"@eslint-community/regexpp@^4.5.1": + version "4.8.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.8.1.tgz#8c4bb756cc2aa7eaf13cfa5e69c83afb3260c20c" + integrity sha512-PWiOzLIUAjN/w5K17PoF4n6sKBw0gqLHPhywmYHP4t1VFQQVYeb1yWsJwnMVEMl3tUHME7X/SJPZLmtG7XBDxQ== + +"@eslint-community/regexpp@^4.6.1": version "4.8.0" resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.8.0.tgz#11195513186f68d42fbf449f9a7136b2c0c92005" integrity sha512-JylOEEzDiOryeUnFbQz+oViCXS0KsvR1mvHkoMiu5+UiBvy+RYX7tzlIIIEstF/gVa2tj9AQXk3dgnxv6KxhFg== @@ -2254,11 +2259,16 @@ "@types/tough-cookie" "*" parse5 "^7.0.0" -"@types/json-schema@*", "@types/json-schema@^7.0.12", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8": +"@types/json-schema@*", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8": version "7.0.12" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== +"@types/json-schema@^7.0.12": + version "7.0.13" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.13.tgz#02c24f4363176d2d18fc8b70b9f3c54aba178a85" + integrity sha512-RbSSoHliUbnXj3ny0CNFOoxrIDV6SUGyStHsvDqosw6CkdPV8TtWGlfecuK4ToyMEAql6pzNxgCFKanovUzlgQ== + "@types/json-stable-stringify@^1.0.32": version "1.0.34" resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.34.tgz#c0fb25e4d957e0ee2e497c1f553d7f8bb668fd75" @@ -2500,9 +2510,9 @@ integrity sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ== "@types/semver@^7.5.0": - version "7.5.1" - resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.1.tgz#0480eeb7221eb9bc398ad7432c9d7e14b1a5a367" - integrity sha512-cJRQXpObxfNKkFAZbJl2yjWtJCqELQIdShsogr1d2MilP8dKD9TE/nEKHkJgUNHdGKCQaf9HbIynuV2csLGVLg== + version "7.5.2" + resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.2.tgz#31f6eec1ed7ec23f4f05608d3a2d381df041f564" + integrity sha512-7aqorHYgdNO4DM36stTiGO3DvKoex9TQRwsJU6vMaFGyqpBA1MNZkz+PG3gaNUPpTAOYhT1WR7M1JyA3fbS9Cw== "@types/send@*": version "0.17.1" @@ -2601,15 +2611,15 @@ "@types/yargs-parser" "*" "@typescript-eslint/eslint-plugin@^6.0.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.6.0.tgz#19ba09aa34fd504696445100262e5a9e1b1d7024" - integrity sha512-CW9YDGTQnNYMIo5lMeuiIG08p4E0cXrXTbcZ2saT/ETE7dWUrNxlijsQeU04qAAKkILiLzdQz+cGFxCJjaZUmA== + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.2.tgz#f18cc75c9cceac8080a9dc2e7d166008c5207b9f" + integrity sha512-ooaHxlmSgZTM6CHYAFRlifqh1OAr3PAQEwi7lhYhaegbnXrnh7CDcHmc3+ihhbQC7H0i4JF0psI5ehzkF6Yl6Q== dependencies: "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "6.6.0" - "@typescript-eslint/type-utils" "6.6.0" - "@typescript-eslint/utils" "6.6.0" - "@typescript-eslint/visitor-keys" "6.6.0" + "@typescript-eslint/scope-manager" "6.7.2" + "@typescript-eslint/type-utils" "6.7.2" + "@typescript-eslint/utils" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" debug "^4.3.4" graphemer "^1.4.0" ignore "^5.2.4" @@ -2618,31 +2628,31 @@ ts-api-utils "^1.0.1" "@typescript-eslint/parser@^6.0.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.6.0.tgz#fe323a7b4eafb6d5ea82b96216561810394a739e" - integrity sha512-setq5aJgUwtzGrhW177/i+DMLqBaJbdwGj2CPIVFFLE0NCliy5ujIdLHd2D1ysmlmsjdL2GWW+hR85neEfc12w== - dependencies: - "@typescript-eslint/scope-manager" "6.6.0" - "@typescript-eslint/types" "6.6.0" - "@typescript-eslint/typescript-estree" "6.6.0" - "@typescript-eslint/visitor-keys" "6.6.0" + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.7.2.tgz#e0ae93771441b9518e67d0660c79e3a105497af4" + integrity sha512-KA3E4ox0ws+SPyxQf9iSI25R6b4Ne78ORhNHeVKrPQnoYsb9UhieoiRoJgrzgEeKGOXhcY1i8YtOeCHHTDa6Fw== + dependencies: + "@typescript-eslint/scope-manager" "6.7.2" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/typescript-estree" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" debug "^4.3.4" -"@typescript-eslint/scope-manager@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.6.0.tgz#57105d4419d6de971f7d2c30a2ff4ac40003f61a" - integrity sha512-pT08u5W/GT4KjPUmEtc2kSYvrH8x89cVzkA0Sy2aaOUIw6YxOIjA8ilwLr/1fLjOedX1QAuBpG9XggWqIIfERw== +"@typescript-eslint/scope-manager@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.7.2.tgz#cf59a2095d2f894770c94be489648ad1c78dc689" + integrity sha512-bgi6plgyZjEqapr7u2mhxGR6E8WCzKNUFWNh6fkpVe9+yzRZeYtDTbsIBzKbcxI+r1qVWt6VIoMSNZ4r2A+6Yw== dependencies: - "@typescript-eslint/types" "6.6.0" - "@typescript-eslint/visitor-keys" "6.6.0" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" -"@typescript-eslint/type-utils@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.6.0.tgz#14f651d13b884915c4fca0d27adeb652a4499e86" - integrity sha512-8m16fwAcEnQc69IpeDyokNO+D5spo0w1jepWWY2Q6y5ZKNuj5EhVQXjtVAeDDqvW6Yg7dhclbsz6rTtOvcwpHg== +"@typescript-eslint/type-utils@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.7.2.tgz#ed921c9db87d72fa2939fee242d700561454f367" + integrity sha512-36F4fOYIROYRl0qj95dYKx6kybddLtsbmPIYNK0OBeXv2j9L5nZ17j9jmfy+bIDHKQgn2EZX+cofsqi8NPATBQ== dependencies: - "@typescript-eslint/typescript-estree" "6.6.0" - "@typescript-eslint/utils" "6.6.0" + "@typescript-eslint/typescript-estree" "6.7.2" + "@typescript-eslint/utils" "6.7.2" debug "^4.3.4" ts-api-utils "^1.0.1" @@ -2651,10 +2661,10 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.62.0.tgz#258607e60effa309f067608931c3df6fed41fd2f" integrity sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ== -"@typescript-eslint/types@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.6.0.tgz#95e7ea650a2b28bc5af5ea8907114a48f54618c2" - integrity sha512-CB6QpJQ6BAHlJXdwUmiaXDBmTqIE2bzGTDLADgvqtHWuhfNP3rAOK7kAgRMAET5rDRr9Utt+qAzRBdu3AhR3sg== +"@typescript-eslint/types@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.7.2.tgz#75a615a6dbeca09cafd102fe7f465da1d8a3c066" + integrity sha512-flJYwMYgnUNDAN9/GAI3l8+wTmvTYdv64fcH8aoJK76Y+1FCZ08RtI5zDerM/FYT5DMkAc+19E4aLmd5KqdFyg== "@typescript-eslint/typescript-estree@5.62.0": version "5.62.0" @@ -2669,30 +2679,30 @@ semver "^7.3.7" tsutils "^3.21.0" -"@typescript-eslint/typescript-estree@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.6.0.tgz#373c420d2e12c28220f4a83352280a04823a91b7" - integrity sha512-hMcTQ6Al8MP2E6JKBAaSxSVw5bDhdmbCEhGW/V8QXkb9oNsFkA4SBuOMYVPxD3jbtQ4R/vSODBsr76R6fP3tbA== +"@typescript-eslint/typescript-estree@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.2.tgz#ce5883c23b581a5caf878af641e49dd0349238c7" + integrity sha512-kiJKVMLkoSciGyFU0TOY0fRxnp9qq1AzVOHNeN1+B9erKFCJ4Z8WdjAkKQPP+b1pWStGFqezMLltxO+308dJTQ== dependencies: - "@typescript-eslint/types" "6.6.0" - "@typescript-eslint/visitor-keys" "6.6.0" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/visitor-keys" "6.7.2" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/utils@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.6.0.tgz#2d686c0f0786da6362d909e27a9de1c13ba2e7dc" - integrity sha512-mPHFoNa2bPIWWglWYdR0QfY9GN0CfvvXX1Sv6DlSTive3jlMTUy+an67//Gysc+0Me9pjitrq0LJp0nGtLgftw== +"@typescript-eslint/utils@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.7.2.tgz#b9ef0da6f04932167a9222cb4ac59cb187165ebf" + integrity sha512-ZCcBJug/TS6fXRTsoTkgnsvyWSiXwMNiPzBUani7hDidBdj1779qwM1FIAmpH4lvlOZNF3EScsxxuGifjpLSWQ== dependencies: "@eslint-community/eslint-utils" "^4.4.0" "@types/json-schema" "^7.0.12" "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "6.6.0" - "@typescript-eslint/types" "6.6.0" - "@typescript-eslint/typescript-estree" "6.6.0" + "@typescript-eslint/scope-manager" "6.7.2" + "@typescript-eslint/types" "6.7.2" + "@typescript-eslint/typescript-estree" "6.7.2" semver "^7.5.4" "@typescript-eslint/visitor-keys@5.62.0": @@ -2703,12 +2713,12 @@ "@typescript-eslint/types" "5.62.0" eslint-visitor-keys "^3.3.0" -"@typescript-eslint/visitor-keys@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.6.0.tgz#1109088b4346c8b2446f3845db526374d9a3bafc" - integrity sha512-L61uJT26cMOfFQ+lMZKoJNbAEckLe539VhTxiGHrWl5XSKQgA0RTBZJW2HFPy5T0ZvPVSD93QsrTKDkfNwJGyQ== +"@typescript-eslint/visitor-keys@6.7.2": + version "6.7.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.2.tgz#4cb2bd786f1f459731b0ad1584c9f73e1c7a4d5c" + integrity sha512-uVw9VIMFBUTz8rIeaUT3fFe8xIUx8r4ywAdlQv1ifH+6acn/XF8Y6rwJ7XNmkNMDrTW+7+vxFFPIF40nJCVsMQ== dependencies: - "@typescript-eslint/types" "6.6.0" + "@typescript-eslint/types" "6.7.2" eslint-visitor-keys "^3.4.1" "@webassemblyjs/ast@1.9.0": @@ -5417,9 +5427,9 @@ eslint-plugin-import@~2.28.0: tsconfig-paths "^3.14.2" eslint-plugin-jsdoc@^46.1.0: - version "46.6.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-46.6.0.tgz#513e41911a3081a15fe6a94490002e8d8f7b71d9" - integrity sha512-T/1gzsvnX45qABzyPEonEhFDttkTn7Igm/X89TXIkTLBOsNl2GYtyBqQPZGXZZ8J5VBzEhiCMvI2P2kXX4dnFw== + version "46.8.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-46.8.1.tgz#cfc649c15d460903fe8e86eda582023bb97f283a" + integrity sha512-uTce7IBluPKXIQMWJkIwFsI1gv7sZRmLjctca2K5DIxPi8fSBj9f4iru42XmGwuiMyH2f3nfc60sFmnSGv4Z/A== dependencies: "@es-joy/jsdoccomment" "~0.40.1" are-docs-informative "^0.0.2"