diff --git a/models.yml b/models.yml
index 0952aa07..70283dd4 100644
--- a/models.yml
+++ b/models.yml
@@ -1905,8 +1905,6 @@ group:
         - agenda_item.can_manage
         - agenda_item.can_see
         - agenda_item.can_see_internal
-        - agenda_item.can_manage_moderator_notes
-        - agenda_item.can_see_moderator_notes
         - assignment.can_manage
         - assignment.can_nominate_other
         - assignment.can_nominate_self
@@ -1915,6 +1913,8 @@ group:
         - list_of_speakers.can_be_speaker
         - list_of_speakers.can_manage
         - list_of_speakers.can_see
+        - list_of_speakers.can_manage_moderator_notes
+        - list_of_speakers.can_see_moderator_notes
         - mediafile.can_manage
         - mediafile.can_see
         - meeting.can_manage_logos_and_fonts
@@ -2111,9 +2111,6 @@ agenda_item:
     description: Given in seconds
     minimum: 0
     restriction_mode: B
-  moderator_notes:
-    type: HTMLStrict
-    restriction_mode: D
   is_internal:
     type: boolean
     description: Calculated by the server
@@ -2188,6 +2185,9 @@ list_of_speakers:
     required: true
     restriction_mode: A
     constant: true
+  moderator_notes:
+    type: HTMLStrict
+    restriction_mode: B
 
   content_object_id:
     type: generic-relation
diff --git a/permission.yml b/permission.yml
index aa703747..f746aa42 100644
--- a/permission.yml
+++ b/permission.yml
@@ -10,9 +10,6 @@ agenda_item:
     can_manage:
         can_see_internal:
             can_see:
-    can_manage_moderator_notes:
-        can_see_moderator_notes:
-            can_see:
 assignment:
     can_manage:
         can_nominate_other:
@@ -25,6 +22,8 @@ list_of_speakers:
     can_manage:
         can_see:
     can_be_speaker:
+    can_manage_moderator_notes:
+        can_see_moderator_notes:
 mediafile:
     can_manage:
         can_see: