Skip to content

Commit

Permalink
Add customizable thumbnails for audio and video attachments (mastodon…
Browse files Browse the repository at this point in the history
…#14145)

- Change audio files to not be stripped of metadata
- Automatically extract cover art from audio if it exists
- Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id`
- Add `icon` to represent it in attachments in ActivityPub
- Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null
- Fix duration of audio not being displayed on public pages until the file is loaded
  • Loading branch information
Gargron authored and Mage committed Jan 14, 2022
1 parent 47bb1c4 commit 289dfe1
Show file tree
Hide file tree
Showing 23 changed files with 247 additions and 138 deletions.
2 changes: 1 addition & 1 deletion app/controllers/api/v1/media_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def check_processing
end

def media_attachment_params
params.permit(:file, :description, :focus)
params.permit(:file, :thumbnail, :description, :focus)
end

def file_type_error
Expand Down
4 changes: 2 additions & 2 deletions app/controllers/media_proxy_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ def show
private

def redownload!
@media_attachment.file_remote_url = @media_attachment.remote_url
@media_attachment.created_at = Time.now.utc
@media_attachment.download_file!
@media_attachment.created_at = Time.now.utc
@media_attachment.save!
end

Expand Down
13 changes: 4 additions & 9 deletions app/controllers/settings/pictures_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,8 @@ class PicturesController < BaseController
before_action :set_picture

def destroy
if valid_picture
account_params = {
@picture => nil,
(@picture + '_remote_url') => nil,
}

msg = UpdateAccountService.new.call(@account, account_params) ? I18n.t('generic.changes_saved_msg') : nil
if valid_picture?
msg = I18n.t('generic.changes_saved_msg') if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' })
redirect_to settings_profile_path, notice: msg, status: 303
else
bad_request
Expand All @@ -30,8 +25,8 @@ def set_picture
@picture = params[:id]
end

def valid_picture
@picture == 'avatar' || @picture == 'header'
def valid_picture?
%w(avatar header).include?(@picture)
end
end
end
3 changes: 2 additions & 1 deletion app/javascript/mastodon/components/status.js
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,8 @@ class Status extends ImmutablePureComponent {
<Component
src={attachment.get('url')}
alt={attachment.get('description')}
poster={status.getIn(['account', 'avatar_static'])}
poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])}
blurhash={attachment.get('blurhash')}
duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
width={this.props.cachedMediaWidth}
height={110}
Expand Down
42 changes: 29 additions & 13 deletions app/javascript/mastodon/features/audio/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ class Audio extends React.PureComponent {
fullscreen: PropTypes.bool,
intl: PropTypes.object.isRequired,
cacheWidth: PropTypes.func,
blurhash: PropTypes.string,
};

state = {
Expand Down Expand Up @@ -222,32 +223,42 @@ class Audio extends React.PureComponent {
window.addEventListener('scroll', this.handleScroll);
window.addEventListener('resize', this.handleResize, { passive: true });

const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = () => this.handlePosterLoad(img);
img.src = this.props.poster;
if (!this.props.blurhash) {
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = () => this.handlePosterLoad(img);
img.src = this.props.poster;
} else {
this._setColorScheme();
this._decodeBlurhash();
}
}

componentDidUpdate (prevProps, prevState) {
if (prevProps.poster !== this.props.poster) {
if (prevProps.poster !== this.props.poster && !this.props.blurhash) {
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = () => this.handlePosterLoad(img);
img.src = this.props.poster;
}

if (prevState.blurhash !== this.state.blurhash) {
const context = this.blurhashCanvas.getContext('2d');
const pixels = decode(this.state.blurhash, 32, 32);
const outputImageData = new ImageData(pixels, 32, 32);

context.putImageData(outputImageData, 0, 0);
if (prevState.blurhash !== this.state.blurhash || prevProps.blurhash !== this.props.blurhash) {
this._setColorScheme();
this._decodeBlurhash();
}

this._clear();
this._draw();
}

_decodeBlurhash () {
const context = this.blurhashCanvas.getContext('2d');
const pixels = decode(this.props.blurhash || this.state.blurhash, 32, 32);
const outputImageData = new ImageData(pixels, 32, 32);

context.putImageData(outputImageData, 0, 0);
}

componentWillUnmount () {
window.removeEventListener('scroll', this.handleScroll);
window.removeEventListener('resize', this.handleResize);
Expand Down Expand Up @@ -415,7 +426,7 @@ class Audio extends React.PureComponent {
}

handlePosterLoad = image => {
const canvas = document.createElement('canvas');
const canvas = document.createElement('canvas');
const context = canvas.getContext('2d');

canvas.width = image.width;
Expand All @@ -425,10 +436,15 @@ class Audio extends React.PureComponent {

const inputImageData = context.getImageData(0, 0, image.width, image.height);
const blurhash = encode(inputImageData.data, image.width, image.height, 4, 4);

this.setState({ blurhash });
}

_setColorScheme () {
const blurhash = this.props.blurhash || this.state.blurhash;
const averageColor = decodeRGB(decode83(blurhash.slice(2, 6)));

this.setState({
blurhash,
color: adjustColor(averageColor),
darkText: luma(averageColor) >= 165,
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ class DetailedStatus extends ImmutablePureComponent {
src={attachment.get('url')}
alt={attachment.get('description')}
duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
poster={status.getIn(['account', 'avatar_static'])}
poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])}
blurhash={attachment.get('blurhash')}
height={150}
/>
);
Expand Down
12 changes: 10 additions & 2 deletions app/lib/activitypub/activity/create.rb
Original file line number Diff line number Diff line change
Expand Up @@ -238,12 +238,13 @@ def process_attachments

begin
href = Addressable::URI.parse(attachment['url']).normalize.to_s
media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
media_attachments << media_attachment

next if unsupported_media_type?(attachment['mediaType']) || skip_download?

media_attachment.file_remote_url = href
media_attachment.download_file!
media_attachment.download_thumbnail!
media_attachment.save
rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
Expand All @@ -256,6 +257,13 @@ def process_attachments
media_attachments
end

def icon_url_from_attachment(attachment)
url = attachment['icon'].is_a?(Hash) ? attachment['icon']['url'] : attachment['icon']
Addressable::URI.parse(url).normalize.to_s if url.present?
rescue Addressable::URI::InvalidURIError
nil
end

def process_poll
return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array))

Expand Down
29 changes: 14 additions & 15 deletions app/models/concerns/remotable.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ module Remotable
extend ActiveSupport::Concern

class_methods do
def remotable_attachment(attachment_name, limit, suppress_errors: true)
attribute_name = "#{attachment_name}_remote_url".to_sym
method_name = "#{attribute_name}=".to_sym
alt_method_name = "reset_#{attachment_name}!".to_sym
def remotable_attachment(attachment_name, limit, suppress_errors: true, download_on_assign: true, attribute_name: nil)
attribute_name ||= "#{attachment_name}_remote_url".to_sym

define_method("download_#{attachment_name}!") do
url = self[attribute_name]

define_method method_name do |url|
return if url.blank?

begin
Expand All @@ -18,7 +18,7 @@ def remotable_attachment(attachment_name, limit, suppress_errors: true)
return
end

return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?)
return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank?

begin
Request.new(:get, url).perform do |response|
Expand All @@ -36,10 +36,8 @@ def remotable_attachment(attachment_name, limit, suppress_errors: true)

basename = SecureRandom.hex(8)

send("#{attachment_name}_file_name=", basename + extname)
send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))

self[attribute_name] = url if has_attribute?(attribute_name)
public_send("#{attachment_name}_file_name=", basename + extname)
public_send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
end
rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
Expand All @@ -50,14 +48,15 @@ def remotable_attachment(attachment_name, limit, suppress_errors: true)
end
end

define_method alt_method_name do
url = self[attribute_name]
define_method("#{attribute_name}=") do |url|
return if self[attribute_name] == url && public_send("#{attachment_name}_file_name").present?

return if url.blank?
self[attribute_name] = url

self[attribute_name] = ''
send(method_name, url)
public_send("download_#{attachment_name}!") if download_on_assign
end

alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!")
end
end

Expand Down
Loading

0 comments on commit 289dfe1

Please sign in to comment.