Merge remote-tracking branch 'origin/release-v1.8.0' into dinsic-rele…

…ase-v1.8.x * origin/release-v1.8.0: (27 commits) Test for outliers whose auth events are in a different room Test for receiving events with auth events in the wrong room enable frozen dicts in synapse (#778) Fix c&p error in #779 Add support for synapse multi DB setups. (#775) Add console feedbak for the TAP output mode (#779) workaround for matrix-org/synapse#6536 (#772) Test for backfilled events whose prev_events cross room boundaries (#769) Add dendrite's logs to buildkite's artifacts (#773) Make the replication torture level adjustable (#771) Add test for removing redactions from search results (#747) Sanity-checking of some params (#765) Fix some 'undefined value' warnings (#768) Remove redundant fallback in on_request_federation_v1_event (#766) Deflake an AS test (#764) Add tests for outbound v2 invite API Fix diagnostics from logout test (#763) Add comment to explain what the v1 outbound test relies on Re-fix test names Fix test names ...
matrix-org · Mar 20, 2020 · a310779 · a310779
2 parents 3c3a0bd + c01db90
commit a310779
Show file tree

Hide file tree

Showing 18 changed files with 1,071 additions and 276 deletions.
diff --git a/lib/SyTest/Federation/Protocol.pm b/lib/SyTest/Federation/Protocol.pm
@@ -26,6 +26,7 @@ use Protocol::Matrix qw(
    encode_base64_unpadded
 );
 
+use Carp;
 
 use Exporter 'import';
 our @EXPORT_OK = qw(
@@ -44,6 +45,7 @@ Calculates the reference hash of an event.
 sub hash_event
 {
    my ( $event ) = @_;
+   croak "Require an event" unless ref $event eq 'HASH';
    my $redacted = redacted_event( $event );
    delete $redacted->{signatures};
    delete $redacted->{age_ts};

diff --git a/lib/SyTest/Federation/Room.pm b/lib/SyTest/Federation/Room.pm
@@ -385,14 +385,21 @@ sub make_join_protoevent
       $self->get_current_state_event( "m.room.create" ),
       $self->get_current_state_event( "m.room.join_rules" ),
    );
+   my $auth_events = $self->make_event_refs( @auth_events );
+
+   my @prev_events = @{ $self->{prev_events} };
+   for my $ev ( @prev_events ) {
+      die "undefined prev_event" if not defined $ev;
+   }
+   my $prev_events = $self->make_event_refs( @prev_events );
 
    return {
       type => "m.room.member",
 
-      auth_events      => $self->make_event_refs( @auth_events ),
+      auth_events      => $auth_events,
       content          => { membership => "join" },
       depth            => JSON::number($self->next_depth),
-      prev_events      => $self->make_event_refs( @{ $self->{prev_events} } ),
+      prev_events      => $prev_events,
       room_id          => $self->room_id,
       sender           => $user_id,
       state_key        => $user_id,
@@ -416,6 +423,8 @@ sub id_for_event
    my $self = shift;
    my ( $event ) = @_;
 
+   croak "Require an event" unless ref $event eq 'HASH';
+
    return SyTest::Federation::Protocol::id_for_event(
       $event, $self->room_version,
    );

diff --git a/lib/SyTest/Federation/Server.pm b/lib/SyTest/Federation/Server.pm
@@ -253,10 +253,7 @@ sub on_request_federation_v1_event
    my $self = shift;
    my ( $req, $event_id ) = @_;
 
-   my $event = $self->{datastore}->get_event( $event_id ) or
-      return Future->done( response => HTTP::Response->new(
-         404, "Not found", [ Content_length => 0 ], "",
-      ) );
+   my $event = $self->{datastore}->get_event( $event_id );
 
    Future->done( json => {
       origin           => $self->server_name,
@@ -285,6 +282,18 @@ sub on_request_federation_v1_make_join
 }
 
 sub on_request_federation_v1_send_join
+{
+   my $self = shift;
+
+   $self->on_request_federation_v2_send_join( @_ )->then( sub {
+      my $res = @_;
+
+      # /v1/send_join has an extraneous [ 200, ... ] wrapper (see MSC1802)
+      Future->done( json => [ 200, $res ] );
+   })
+}
+
+sub on_request_federation_v2_send_join
 {
    my $self = shift;
    my ( $req, $room_id ) = @_;
@@ -305,20 +314,19 @@ sub on_request_federation_v1_send_join
 
    $room->insert_event( $event );
 
-   # /v1/send_join has an extraneous [ 200, ... ] wrapper (see MSC1802)
-   Future->done( json => [ 200, {
+   Future->done( json => {
       auth_chain => \@auth_chain,
       state      => \@state_events,
-   } ] );
+   } );
 }
 
 sub mk_await_request_pair
 {
    my $class = shift;
-   my ( $shortname, $paramnames ) = @_;
+   my ( $versionprefix, $shortname, $paramnames ) = @_;
    my @paramnames = @$paramnames;
 
-   my $okey = "awaiting_$shortname";
+   my $okey = "awaiting_${versionprefix}_${shortname}";
 
    my $awaitfunc = sub {
       my $self = shift;
@@ -336,7 +344,9 @@ sub mk_await_request_pair
          });
    };
 
-   my $was_on_requestfunc = $class->can( "on_request_federation_v1_$shortname" );
+   my $was_on_requestfunc = $class->can(
+      "on_request_federation_${versionprefix}_${shortname}"
+   );
    my $on_requestfunc = sub {
       my $self = shift;
       my ( $req, @pathvalues ) = @_;
@@ -375,64 +385,74 @@ sub mk_await_request_pair
 
    no strict 'refs';
    no warnings 'redefine';
-   *{"${class}::await_request_$shortname"} = $awaitfunc;
-   *{"${class}::on_request_federation_v1_$shortname"} = $on_requestfunc;
+   *{"${class}::await_request_${versionprefix}_${shortname}"} = $awaitfunc;
+   # Deprecated alternative name for v1 endpoints.
+   *{"${class}::await_request_${shortname}"} = $awaitfunc if ${versionprefix} eq "v1";
+   *{"${class}::on_request_federation_${versionprefix}_${shortname}"} = $on_requestfunc;
 }
 
 __PACKAGE__->mk_await_request_pair(
-   query_directory => [qw( ?room_alias )],
+   "v1", "query_directory", [qw( ?room_alias )],
+);
+
+__PACKAGE__->mk_await_request_pair(
+   "v1", "query_profile", [qw( ?user_id )],
+);
+
+__PACKAGE__->mk_await_request_pair(
+   "v1", "make_join", [qw( :room_id :user_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   query_profile => [qw( ?user_id )],
+   "v1", "make_leave", [qw( :room_id :user_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   make_join => [qw( :room_id :user_id )],
+   "v1", "send_join", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   make_leave => [qw( :room_id :user_id )],
+   "v2", "send_join", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   send_join => [qw( :room_id )],
+   "v1", "state_ids", [qw( :room_id ?event_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   state_ids => [qw( :room_id ?event_id )],
+   "v1", "state", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   state => [qw( :room_id )],
+   "v1", "get_missing_events", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   get_missing_events => [qw( :room_id )],
+   "v1", "event_auth", [qw( :room_id :event_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   event_auth => [qw( :room_id :event_id )],
+   "v1", "backfill", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   backfill => [qw( :room_id )],
+   "v1", "invite", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   invite => [qw( :room_id )],
+   "v2", "invite", [qw( :room_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   event => [qw( :event_id )],
+   "v1", "event", [qw( :event_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   user_devices => [qw( :user_id )],
+   "v1", "user_devices", [qw( :user_id )],
 );
 
 __PACKAGE__->mk_await_request_pair(
-   user_keys_query => [qw( )],
+   "v1", "user_keys_query", [qw( )],
 );
 
 sub on_request_federation_v1_send

diff --git a/lib/SyTest/Homeserver.pm b/lib/SyTest/Homeserver.pm
@@ -296,6 +296,10 @@ It returns the config hash.
 
 =cut
 
+# This is now only used for Dendrite, as synapse uses `_get_dbconfigs`. It
+# probably makes more sense to have a dendrite specific handling, rather than
+# using synapse config format and then parsing and converting it into dendrite
+# config.
 sub _get_dbconfig
 {
    my $self = shift;
@@ -340,6 +344,85 @@ sub _get_dbconfig
    return %db_config;
 }
 
+
+=head2 _get_dbconfigs
+
+   %db_configs = $self->_get_dbconfigs( %defaults )
+
+This method loads the database configs from either C<databases.yaml> or
+C<database.yaml>, using the default if it doesn't exist.
+
+It then passes the loaded configs to C<_check_db_config> for
+sanity-checking. That method may be overridden by subclasses, and should C<die>
+if there is a problem with the config.
+
+Finally, it calls the relevant clear_db method on each database to clear out
+the configured databases.
+
+It returns the configs as a hash of database name to config hashref.
+
+=cut
+
+sub _get_dbconfigs
+{
+   my $self = shift;
+   my ( %defaults ) = @_;
+
+   my $hs_dir = $self->{hs_dir};
+
+   # Try and load the configs from the various locations.
+   my ( %db_configs );
+   if ( -f "$hs_dir/databases.yaml") {
+      $self->{output}->diag( "Using DB config from $hs_dir/databases.yaml" );
+
+      %db_configs = %{ YAML::LoadFile( "$hs_dir/databases.yaml" ) };
+   }
+   elsif( -f "$hs_dir/database.yaml" ) {
+      $self->{output}->diag( "Using DB config from $hs_dir/database.yaml" );
+
+      my %db_config = %{ YAML::LoadFile( "$hs_dir/database.yaml" ) };
+
+      $db_configs{"main"} = \%db_config;
+   }
+   else {
+      $self->{output}->diag( "Using default DB config and writing to $hs_dir/database.yaml" );
+
+      YAML::DumpFile( "$hs_dir/database.yaml", \%defaults );
+      $db_configs{"main"} = \%defaults;
+   }
+
+   # Go through each database and check the config and clear the database.
+   foreach my $db ( keys %db_configs ) {
+      my $db_config = $db_configs{$db};
+
+      # backwards-compatibility hacks
+      my $db_name = delete $db_config->{name};
+      if( defined $db_name ) {
+         if( $db_name eq 'psycopg2' ) {
+            $db_config->{type} = 'pg';
+         }
+         elsif( $db_name eq 'sqlite3' ) {
+            $db_config->{type} = 'sqlite';
+         }
+         else {
+            die "Unrecognised DB name '$db_name'";
+         }
+      }
+
+      eval {
+         $self->_check_db_config( %{ $db_config } );
+         1;
+      } or die "Error loading db config: $@";
+
+      my $db_type = $db_config->{type};
+      my $clear_meth = "_clear_db_${db_type}";
+      $self->$clear_meth( %{ $db_config->{args} } );
+   }
+
+   return %db_configs;
+}
+
+
 sub _check_db_config
 {
    my $self = shift;

diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm
@@ -106,29 +106,26 @@ sub start
 
    my $hs_dir = $self->{hs_dir};
 
-   my %db_config = $self->_get_dbconfig(
+   my %db_configs = $self->_get_dbconfigs(
       type => 'sqlite',
       args => {
          database => ":memory:", #"$hs_dir/homeserver.db",
       },
    );
 
-   my $db_type = $db_config{type};
+   # convert sytest db args onto synapse db args
+   for my $db ( keys %db_configs ) {
+      my %db_config = %{ $db_configs{$db} };
 
-   # map sytest db args onto synapse db args
-   my %synapse_db_config;
-   if( $db_type eq "pg" ) {
-      %synapse_db_config = (
-         name => 'psycopg2',
-         args => $db_config{args},
-      );
-   }
-   else {
-      # must be sqlite
-      %synapse_db_config = (
-         name => 'sqlite3',
-         args => $db_config{args},
-      );
+      my $db_type = $db_config{type};
+
+      if( $db_type eq "pg" ) {
+         $db_configs{$db}{name} = 'psycopg2';
+      }
+      else {
+         # must be sqlite
+         $db_configs{$db}{name} = 'sqlite3';
+      }
    }
 
    # Clean up the media_store directory each time, or else it fills up with
@@ -214,13 +211,13 @@ sub start
         },
 
         enable_registration => "true",
-        database => \%synapse_db_config,
+        databases => \%db_configs,
         macaroon_secret_key => $macaroon_secret_key,
         registration_shared_secret => $registration_shared_secret,
 
         pid_file => "$hs_dir/homeserver.pid",
 
-        use_frozen_events => "true",
+        use_frozen_dicts => "true",
 
         allow_guest_access => "True",
 
@@ -659,11 +656,12 @@ sub _init
    $self->SUPER::_init( @_ );
 
    $self->{dendron} = delete $args->{dendron_binary};
-   if( delete $args->{torture_replication} ) {
+
+   if( my $level = delete $args->{torture_replication} ) {
       # torture the replication protocol a bit, to replicate bugs.
       # (value is the number of ms to wait before sending out each batch of
       # updates.)
-      $self->{replication_torture_level} = 50;
+      $self->{replication_torture_level} = $level;
    }
 
    my $idx = $self->{hs_index};

diff --git a/lib/SyTest/HomeserverFactory/Synapse.pm b/lib/SyTest/HomeserverFactory/Synapse.pm
@@ -109,7 +109,7 @@ sub get_options
 
    return (
       'dendron-binary=s' => \$self->{args}{dendron_binary},
-      'torture-replication+' => \$self->{args}{torture_replication},
+      'torture-replication:50' => \$self->{args}{torture_replication},
       $self->SUPER::get_options(),
    );
 }
@@ -124,7 +124,7 @@ sub print_usage
 
        --dendron-binary PATH    - path to the 'dendron' binary
 
-       --torture-replication    - enable torturing of the replication protocol
+       --torture-replication[=LEVEL] - enable torturing of the replication protocol
 EOF
 }