event,bridgev2: implement MSC4231

bridge/matrix.go

@@ -604,6 +604,10 @@ func (mx *MatrixHandler) HandleMessage(ctx context.Context, evt *event.Event) {
 	}
 
 	content := evt.Content.AsMessage()
+	if content.IsCaptionFallback() {
+		mx.bridge.SendMessageCheckpoint(evt, status.MsgStepRemote, fmt.Errorf("ignoring caption fallback"), status.MsgStatusUnsupported, 0)
+		return
+	}
 	content.RemoveReplyFallback()
 	if user.GetPermissionLevel() >= bridgeconfig.PermissionLevelUser && content.MsgType == event.MsgText {
 		commandPrefix := mx.bridge.Config.Bridge.GetCommandPrefix()

bridgev2/bridgeconfig/config.go

@@ -79,6 +79,7 @@ type MatrixConfig struct {
 	MessageStatusEvents bool  `yaml:"message_status_events"`
 	DeliveryReceipts    bool  `yaml:"delivery_receipts"`
 	MessageErrorNotices bool  `yaml:"message_error_notices"`
+	CaptionFallbacks    bool  `yaml:"caption_fallbacks"`
 	SyncDirectChatList  bool  `yaml:"sync_direct_chat_list"`
 	FederateRooms       bool  `yaml:"federate_rooms"`
 	UploadFileThreshold int64 `yaml:"upload_file_threshold"`

bridgev2/bridgeconfig/upgrade.go

@@ -86,6 +86,7 @@ func doUpgrade(helper up.Helper) {
 	helper.Copy(up.Bool, "matrix", "message_status_events")
 	helper.Copy(up.Bool, "matrix", "delivery_receipts")
 	helper.Copy(up.Bool, "matrix", "message_error_notices")
+	helper.Copy(up.Bool, "matrix", "caption_fallbacks")
 	helper.Copy(up.Bool, "matrix", "sync_direct_chat_list")
 	helper.Copy(up.Bool, "matrix", "federate_rooms")
 	helper.Copy(up.Int, "matrix", "upload_file_threshold")

bridgev2/matrix/intent.go

@@ -44,7 +44,45 @@ type ASIntent struct {
 var _ bridgev2.MatrixAPI = (*ASIntent)(nil)
 var _ bridgev2.MarkAsDMMatrixAPI = (*ASIntent)(nil)
 
-func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType event.Type, content *event.Content, extra *bridgev2.MatrixSendExtra) (*mautrix.RespSendEvent, error) {
+func (as *ASIntent) sendCaptionFallback(ctx context.Context, roomID id.RoomID, content *event.MessageEventContent, ts time.Time, mainEvt id.EventID) {
+	captionFallbackContent := &event.MessageEventContent{
+		MsgType:       event.MsgText,
+		Body:          content.Body,
+		Format:        content.Format,
+		FormattedBody: content.FormattedBody,
+		Mentions:      content.Mentions,
+		RelatesTo: &event.RelatesTo{
+			InReplyTo: &event.InReplyTo{
+				EventID: mainEvt,
+			},
+		},
+		//StableIsCaptionFallback:   true,
+		UnstableIsCaptionFallback: true,
+	}
+	if content.RelatesTo != nil && content.RelatesTo.Type == event.RelThread {
+		captionFallbackContent.RelatesTo.Type = event.RelThread
+		captionFallbackContent.RelatesTo.EventID = content.RelatesTo.EventID
+	}
+	var err error
+	var resp *mautrix.RespSendEvent
+	if ts.IsZero() {
+		resp, err = as.Matrix.SendMessageEvent(ctx, roomID, event.EventMessage, captionFallbackContent)
+	} else {
+		resp, err = as.Matrix.SendMassagedMessageEvent(ctx, roomID, event.EventMessage, captionFallbackContent, ts.UnixMilli())
+	}
+	if err != nil {
+		zerolog.Ctx(ctx).Warn().Err(err).
+			Stringer("main_message_event_id", mainEvt).
+			Msg("Failed to send caption fallback")
+	} else {
+		zerolog.Ctx(ctx).Debug().
+			Stringer("main_message_event_id", mainEvt).
+			Stringer("caption_fallback_event_id", resp.EventID).
+			Msg("Sent caption fallback")
+	}
+}
+
+func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType event.Type, content *event.Content, extra *bridgev2.MatrixSendExtra) (resp *mautrix.RespSendEvent, err error) {
 	if extra == nil {
 		extra = &bridgev2.MatrixSendExtra{}
 	}
@@ -57,6 +95,13 @@ func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType
 			Extra:  content.Raw,
 		})
 	}
+	if as.Connector.Config.Matrix.CaptionFallbacks && eventType == event.EventMessage {
+		if msgContent, ok := content.Parsed.(*event.MessageEventContent); ok && msgContent.HasCaption() {
+			defer func() {
+				go as.sendCaptionFallback(ctx, roomID, msgContent, extra.Timestamp, resp.EventID)
+			}()
+		}
+	}
 	if eventType != event.EventReaction && eventType != event.EventRedaction {
 		if encrypted, err := as.Matrix.StateStore.IsEncrypted(ctx, roomID); err != nil {
 			return nil, fmt.Errorf("failed to check if room is encrypted: %w", err)

bridgev2/matrix/mxmain/example-config.yaml

@@ -210,6 +210,8 @@ matrix:
     delivery_receipts: false
     # Whether the bridge should send error notices via m.notice events when a message fails to bridge.
     message_error_notices: true
+    # Should the bridge send fallbacks for media captions as per MSC4231?
+    caption_fallbacks: false
     # Whether the bridge should update the m.direct account data event when double puppeting is enabled.
     sync_direct_chat_list: true
     # Whether created rooms should have federation enabled. If false, created portal rooms

bridgev2/messagestatus.go

@@ -113,6 +113,11 @@ func (ms MessageStatus) WithErrorAsMessage() MessageStatus {
 	return ms
 }
 
+func (ms MessageStatus) WithoutMSS() MessageStatus {
+	ms.DisableMSS = true
+	return ms
+}
+
 func (ms MessageStatus) Error() string {
 	return ms.InternalError.Error()
 }

bridgev2/queue.go

@@ -98,6 +98,15 @@ func (br *Bridge) QueueMatrixEvent(ctx context.Context, evt *event.Event) {
 	}
 	if evt.Type == event.EventMessage && sender != nil {
 		msg := evt.Content.AsMessage()
+		if msg.IsCaptionFallback() {
+			status := WrapErrorInStatus(errors.New("ignoring caption fallback")).
+				WithErrorReason(event.MessageStatusUnsupported).
+				WithIsCertain(true).
+				WithErrorAsMessage().
+				WithoutMSS()
+			br.Matrix.SendMessageStatus(ctx, &status, StatusEventInfoFromEvent(evt))
+			return
+		}
 		msg.RemoveReplyFallback()
 		if strings.HasPrefix(msg.Body, br.Config.CommandPrefix) || evt.RoomID == sender.ManagementRoom {
 			if !sender.Permissions.Commands {

event/message.go

@@ -128,7 +128,9 @@ type MessageEventContent struct {
 	FromDevice id.DeviceID          `json:"from_device,omitempty"`
 	Methods    []VerificationMethod `json:"methods,omitempty"`
 
-	replyFallbackRemoved bool
+	StableIsCaptionFallback   bool `json:"m.caption_fallback,omitempty"`
+	UnstableIsCaptionFallback bool `json:"org.matrix.msc4231.caption_fallback,omitempty"`
+	replyFallbackRemoved      bool
 
 	MessageSendRetry         *BeeperRetryMetadata     `json:"com.beeper.message_send_retry,omitempty"`
 	BeeperGalleryImages      []*MessageEventContent   `json:"com.beeper.gallery.images,omitempty"`
@@ -142,6 +144,11 @@ type MessageEventContent struct {
 	MSC3245Voice *MSC3245Voice `json:"org.matrix.msc3245.voice,omitempty"`
 }
 
+func (content *MessageEventContent) IsCaptionFallback() bool {
+	return content != nil && (content.StableIsCaptionFallback || content.UnstableIsCaptionFallback ||
+		(content.NewContent != nil && (content.NewContent.StableIsCaptionFallback || content.NewContent.UnstableIsCaptionFallback)))
+}
+
 func (content *MessageEventContent) GetFileName() string {
 	if content.FileName != "" {
 		return content.FileName
@@ -208,6 +215,10 @@ func ReverseTextToHTML(input string) string {
 	return html.UnescapeString(strings.ReplaceAll(input, "<br/>", "\n"))
 }
 
+func (content *MessageEventContent) HasCaption() bool {
+	return content.MsgType.IsMedia() && content.Body != "" && content.FileName != "" && content.Body != content.FileName
+}
+
 func (content *MessageEventContent) EnsureHasHTML() {
 	if len(content.FormattedBody) == 0 || content.Format != FormatHTML {
 		content.FormattedBody = TextToHTML(content.Body)