[federation] Implement get missing events api (#516)

* [federation] implement get_missing_events

* Check that events are viewable by the server

* Explain filterEvents

src/github.com/matrix-org/dendrite/federationapi/routing/missingevents.go

@@ -0,0 +1,80 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type getMissingEventRequest struct {
+	EarliestEvents []string `json:"earliest_events"`
+	LatestEvents   []string `json:"latest_events"`
+	Limit          int      `json:"limit"`
+	MinDepth       int64    `json:"min_depth"`
+}
+
+// GetMissingEvents returns missing events between earliest_events & latest_events.
+// Events are fetched from room DAG starting from latest_events until we reach earliest_events or the limit.
+func GetMissingEvents(
+	httpReq *http.Request,
+	request *gomatrixserverlib.FederationRequest,
+	query api.RoomserverQueryAPI,
+	roomID string,
+) util.JSONResponse {
+	var gme getMissingEventRequest
+	if err := json.Unmarshal(request.Content(), &gme); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()),
+		}
+	}
+
+	var eventsResponse api.QueryMissingEventsResponse
+	if err := query.QueryMissingEvents(
+		httpReq.Context(), &api.QueryMissingEventsRequest{
+			EarliestEvents: gme.EarliestEvents,
+			LatestEvents:   gme.LatestEvents,
+			Limit:          gme.Limit,
+			ServerName:     request.Origin(),
+		},
+		&eventsResponse,
+	); err != nil {
+		return httputil.LogThenError(httpReq, err)
+	}
+
+	eventsResponse.Events = filterEvents(eventsResponse.Events, gme.MinDepth, roomID)
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: eventsResponse,
+	}
+}
+
+// filterEvents returns only those events with matching roomID and having depth greater than minDepth
+func filterEvents(
+	events []gomatrixserverlib.Event, minDepth int64, roomID string,
+) []gomatrixserverlib.Event {
+	ref := events[:0]
+	for _, ev := range events {
+		if ev.Depth() >= minDepth && ev.RoomID() == roomID {
+			ref = append(ref, ev)
+		}
+	}
+	return ref
+}

src/github.com/matrix-org/dendrite/federationapi/routing/routing.go

@@ -176,4 +176,12 @@ func Setup(
 			return Version()
 		},
 	)).Methods(http.MethodGet)
+
+	v1fedmux.Handle("get_missing_events/{roomID}", common.MakeFedAPI(
+		"federation_get_missing_events", cfg.Matrix.ServerName, keys,
+		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
+			vars := mux.Vars(httpReq)
+			return GetMissingEvents(httpReq, request, query, vars["roomID"])
+		},
+	)).Methods(http.MethodGet)
 }

src/github.com/matrix-org/dendrite/roomserver/api/query.go

@@ -154,6 +154,24 @@ type QueryServerAllowedToSeeEventResponse struct {
 	AllowedToSeeEvent bool `json:"can_see_event"`
 }
 
+// QueryMissingEventsRequest is a request to QueryMissingEvents
+type QueryMissingEventsRequest struct {
+	// Events which are known previous to the gap in the timeline.
+	EarliestEvents []string `json:"earliest_events"`
+	// Latest known events.
+	LatestEvents []string `json:"latest_events"`
+	// Limit the number of events this query returns.
+	Limit int `json:"limit"`
+	// The server interested in the event
+	ServerName gomatrixserverlib.ServerName `json:"server_name"`
+}
+
+// QueryMissingEventsResponse is a response to QueryMissingEvents
+type QueryMissingEventsResponse struct {
+	// Missing events, arbritrary order.
+	Events []gomatrixserverlib.Event `json:"events"`
+}
+
 // QueryStateAndAuthChainRequest is a request to QueryStateAndAuthChain
 type QueryStateAndAuthChainRequest struct {
 	// The room ID to query the state in.
@@ -225,6 +243,13 @@ type RoomserverQueryAPI interface {
 		response *QueryServerAllowedToSeeEventResponse,
 	) error
 
+	// Query missing events for a room from roomserver
+	QueryMissingEvents(
+		ctx context.Context,
+		request *QueryMissingEventsRequest,
+		response *QueryMissingEventsResponse,
+	) error
+
 	// Query to get state and auth chain for a (potentially hypothetical) event.
 	// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate
 	// the state and auth chain to return.
@@ -253,6 +278,9 @@ const RoomserverQueryInvitesForUserPath = "/api/roomserver/queryInvitesForUser"
 // RoomserverQueryServerAllowedToSeeEventPath is the HTTP path for the QueryServerAllowedToSeeEvent API
 const RoomserverQueryServerAllowedToSeeEventPath = "/api/roomserver/queryServerAllowedToSeeEvent"
 
+// RoomserverQueryMissingEventsPath is the HTTP path for the QueryMissingEvents API
+const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents"
+
 // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API
 const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain"
 
@@ -348,6 +376,19 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent(
 	return postJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+// QueryMissingEvents implements RoomServerQueryAPI
+func (h *httpRoomserverQueryAPI) QueryMissingEvents(
+	ctx context.Context,
+	request *QueryMissingEventsRequest,
+	response *QueryMissingEventsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingEvents")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath
+	return postJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryStateAndAuthChain implements RoomserverQueryAPI
 func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain(
 	ctx context.Context,

src/github.com/matrix-org/dendrite/roomserver/auth/auth.go

@@ -0,0 +1,47 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import "github.com/matrix-org/gomatrixserverlib"
+
+// IsServerAllowed returns true if there exists a event in authEvents
+// which allows server to view this event. That is true when a client on the server
+// can view the event. Otherwise returns false.
+func IsServerAllowed(
+	serverName gomatrixserverlib.ServerName,
+	authEvents []gomatrixserverlib.Event,
+) bool {
+	for _, ev := range authEvents {
+		membership, err := ev.Membership()
+		if err != nil || membership != "join" {
+			continue
+		}
+
+		stateKey := ev.StateKey()
+		if stateKey == nil {
+			continue
+		}
+
+		_, domain, err := gomatrixserverlib.SplitID('@', *stateKey)
+		if err != nil {
+			continue
+		}
+
+		if domain == serverName {
+			return true
+		}
+	}
+
+	// TODO: Check if history visibility is shared and if the server is currently in the room
+	return false
+}

src/github.com/matrix-org/dendrite/roomserver/query/query.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -374,58 +375,85 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent(
 	ctx context.Context,
 	request *api.QueryServerAllowedToSeeEventRequest,
 	response *api.QueryServerAllowedToSeeEventResponse,
-) error {
+) (err error) {
-	stateEntries, err := state.LoadStateAtEvent(ctx, r.DB, request.EventID)
+	response.AllowedToSeeEvent, err = r.checkServerAllowedToSeeEvent(
+		ctx, request.EventID, request.ServerName,
+	)
+	return
+}
+
+func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent(
+	ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName,
+) (bool, error) {
+	stateEntries, err := state.LoadStateAtEvent(ctx, r.DB, eventID)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	// TODO: We probably want to make it so that we don't have to pull
 	// out all the state if possible.
 	stateAtEvent, err := r.loadStateEvents(ctx, stateEntries)
+	if err != nil {
+		return false, err
+	}
+
+	return auth.IsServerAllowed(serverName, stateAtEvent), nil
+}
+
+// QueryMissingEvents implements api.RoomserverQueryAPI
+func (r *RoomserverQueryAPI) QueryMissingEvents(
+	ctx context.Context,
+	request *api.QueryMissingEventsRequest,
+	response *api.QueryMissingEventsResponse,
+) error {
+	resultNIDs := make([]types.EventNID, 0, request.Limit)
+	var front []string
+	visited := make(map[string]bool, request.Limit) // request.Limit acts as a hint to size.
+	for _, id := range request.EarliestEvents {
+		visited[id] = true
+	}
+
+	for _, id := range request.LatestEvents {
+		if !visited[id] {
+			front = append(front, id)
+		}
+	}
+
+BFSLoop:
+	for len(front) > 0 {
+		var next []string
+		events, err := r.DB.EventsFromIDs(ctx, front)
 		if err != nil {
 			return err
 		}
 
-	// TODO: Should this be lifted out of here to a more general set of
+		for _, ev := range events {
-	// auth functions?
+			if len(resultNIDs) > request.Limit {
-
+				break BFSLoop
-	isInRoom := false
+			}
-	for _, ev := range stateAtEvent {
+			resultNIDs = append(resultNIDs, ev.EventNID)
-		membership, err := ev.Membership()
+			for _, pre := range ev.PrevEventIDs() {
+				if !visited[pre] {
+					visited[pre] = true
+					allowed, err := r.checkServerAllowedToSeeEvent(
+						ctx, ev.EventID(), request.ServerName,
+					)
 					if err != nil {
-			continue
+						return err
 					}
 
-		if membership != "join" {
+					if allowed {
-			continue
+						next = append(next, pre)
-		}
-
-		stateKey := ev.StateKey()
-		if stateKey == nil {
-			continue
-		}
-
-		_, domain, err := gomatrixserverlib.SplitID('@', *stateKey)
-		if err != nil {
-			continue
-		}
-
-		if domain == request.ServerName {
-			isInRoom = true
-			break
 					}
 				}
-
+			}
-	if isInRoom {
+		}
-		response.AllowedToSeeEvent = true
+		front = next
-		return nil
 	}
 
-	// TODO: Check if history visibility is shared and if the server is currently in the room
+	var err error
-
+	response.Events, err = r.loadEvents(ctx, resultNIDs)
-	response.AllowedToSeeEvent = false
+	return err
-	return nil
 }
 
 // QueryStateAndAuthChain implements api.RoomserverQueryAPI
@@ -607,6 +635,20 @@ func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	servMux.Handle(
+		api.RoomserverQueryMissingEventsPath,
+		common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse {
+			var request api.QueryMissingEventsRequest
+			var response api.QueryMissingEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	servMux.Handle(
 		api.RoomserverQueryStateAndAuthChainPath,
 		common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse {