fix(core): Can() does not work for setting marks (#3223)

Previously, setting marks did no schema validation checks for dry runs
(like the `.can()` command). The `setMark` raw command will now properly
check if the mark is possible to be set given the editor node/mark
schema.

Co-authored-by: Cameron Hessler <cameron.hessler@buildertrend.com>

packages/core/src/commands/setMark.ts

@@ -1,5 +1,7 @@
-import { MarkType } from 'prosemirror-model'
+import { MarkType, ResolvedPos } from 'prosemirror-model'
+import { EditorState, Transaction } from 'prosemirror-state'
 
+import { isTextSelection } from '../helpers'
 import { getMarkAttributes } from '../helpers/getMarkAttributes'
 import { getMarkType } from '../helpers/getMarkType'
 import { RawCommands } from '../types'
@@ -15,6 +17,45 @@ declare module '@tiptap/core' {
   }
 }
 
+function canSetMark(state: EditorState, tr: Transaction, newMarkType: MarkType) {
+  const { selection } = tr
+  let cursor: ResolvedPos | null = null
+
+  if (isTextSelection(selection)) {
+    cursor = selection.$cursor
+  }
+
+  if (cursor) {
+    const currentMarks = state.storedMarks ?? cursor.marks()
+
+    // There can be no current marks that exclude the new mark
+    return !!newMarkType.isInSet(currentMarks) || !currentMarks.some(mark => mark.type.excludes(newMarkType))
+  }
+
+  const { ranges } = selection
+
+  return ranges.some(({ $from, $to }) => {
+    let someNodeSupportsMark = $from.depth === 0 ? state.doc.inlineContent && state.doc.type.allowsMarkType(newMarkType) : false
+
+    state.doc.nodesBetween($from.pos, $to.pos, (node, _pos, parent) => {
+      // If we already found a mark that we can enable, return false to bypass the remaining search
+      if (someNodeSupportsMark) {
+        return false
+      }
+
+      if (node.isInline) {
+        const parentAllowsMarkType = !parent || parent.type.allowsMarkType(newMarkType)
+        const currentMarksAllowMarkType = !!newMarkType.isInSet(node.marks) || !node.marks.some(otherMark => otherMark.type.excludes(newMarkType))
+
+        someNodeSupportsMark = parentAllowsMarkType && currentMarksAllowMarkType
+      }
+      return !someNodeSupportsMark
+    })
+
+    return someNodeSupportsMark
+  })
+
+}
 export const setMark: RawCommands['setMark'] = (typeOrName, attributes = {}) => ({ tr, state, dispatch }) => {
   const { selection } = tr
   const { empty, ranges } = selection
@@ -42,6 +83,7 @@ export const setMark: RawCommands['setMark'] = (typeOrName, attributes = {}) =>
           // we know that we have to merge its attributes
           // otherwise we add a fresh new mark
           if (someHasMark) {
+
             node.marks.forEach(mark => {
               if (type === mark.type) {
                 tr.addMark(trimmedFrom, trimmedTo, type.create({
@@ -58,5 +100,5 @@ export const setMark: RawCommands['setMark'] = (typeOrName, attributes = {}) =>
     }
   }
 
-  return true
+  return canSetMark(state, tr, type)
 }