Foundation for media API testing (#136)
* cmd/mediaapi-integration-tests: Add foundation for testing
* common/test: Add some server init and client request utilities
* common/test/client: Handle timed out requests for tests that passed
* cmd/syncserver-integration-tests: Port to new common/test infra
* common/test/client: Remove stray debug print
* cmd/mediaapi-integration-tests: Simplify slice initialisation
* cmd/mediaapi-integration-tests: Simplify getMediaURL argument
* cmd/mediaapi-integration-tests: Make startMediaAPI return listen address
* common/test/client: Fix uninitialised LastRequestErr
* common/test/server: Remove redundant argument
* common/test/server: Add StartProxy to create a reverse proxy
* cmd/mediaapi-integration-tests: Add proxies in front of servers
This is needed so that origins can be correctly configured and used for
remote media.
* travis: Enable media API integration tests
* travis: Build the client-api-proxy for media tests
* common/test/client: Don't panic on EOF in CanonicalJSONInput
* cmd/mediaapi-integration-tests: Add upload/download/thumbnail tests
* mediaapi/thumbnailer: Store thumbnail according to requested size
* cmd/mediaapi-integration-tests: Add totem.jpg test file
* cmd/client-api-proxy: Optionally listen for HTTPS
* common/test/client: Do not verify TLS certs for testing
We will commonly use self-signed certs.
* cmd/mediaapi-integration-tests: Make HTTPS requests
* cmd/mediaapi-integration-tests: Log size and method for thumbnails
* mediaapi/thumbnailer: Factor out isThumbnailExists
Appease gocyclo^w^w simplify
* mediaapi/thumbnailer: Check if request is larger than original
* travis: Install openssl and generate server.{crt,key}
* cmd/mediaapi-integration-tests: Add valid dynamic thumbnail test
* cmd/mediaapi-integration-tests: Document state of tests
* cmd/mediaapi-integration-tests: Test remote thumbnail before download
This ordering also exercises the cold cache immediate generation of a
size configured for pregeneration.
* travis: Explain openssl key+cert generation
* common/test/server: Clarify postgresContainerName
This commit is contained in:
Robert Swain
GitHub
parent
b184a48897
commit
6eae6f7598
@ -8,6 +8,9 @@ sudo: false
|
||||
dist: trusty
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- openssl
|
||||
postgresql: "9.5"
|
||||
|
||||
services:
|
||||
@ -18,6 +21,10 @@ install:
|
||||
- go get github.com/golang/lint/golint
|
||||
- go get github.com/fzipp/gocyclo
|
||||
|
||||
# Generate a self-signed X.509 certificate for TLS.
|
||||
before_script:
|
||||
- openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost
|
||||
|
||||
script:
|
||||
- ./travis-install-kafka.sh
|
||||
- ./travis-test.sh
|
||||
@ -29,4 +36,3 @@ notifications:
|
||||
on_success: change # always|never|change
|
||||
on_failure: always
|
||||
on_start: never
|
||||
|
||||
|
||||
@ -51,6 +51,8 @@ var (
|
||||
clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
|
||||
mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
|
||||
bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.")
|
||||
certFile = flag.String("tls-cert", "server.crt", "The PEM formatted X509 certificate to use for TLS")
|
||||
keyFile = flag.String("tls-key", "server.key", "The PEM private key to use for TLS")
|
||||
)
|
||||
|
||||
func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
|
||||
@ -148,6 +150,9 @@ func main() {
|
||||
fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1")
|
||||
fmt.Println(" /* => ", *clientAPIURL+"/api/*")
|
||||
fmt.Println("Listening on ", *bindAddress)
|
||||
srv.ListenAndServe()
|
||||
|
||||
if *certFile != "" && *keyFile != "" {
|
||||
panic(srv.ListenAndServeTLS(*certFile, *keyFile))
|
||||
} else {
|
||||
panic(srv.ListenAndServe())
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@ -36,27 +37,29 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
bindAddr = os.Getenv("BIND_ADDRESS")
|
||||
bindAddr = flag.String("listen", "", "The port to listen on.")
|
||||
dataSource = os.Getenv("DATABASE")
|
||||
logDir = os.Getenv("LOG_DIR")
|
||||
serverName = os.Getenv("SERVER_NAME")
|
||||
basePath = os.Getenv("BASE_PATH")
|
||||
// Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited
|
||||
maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES")
|
||||
configPath = os.Getenv("CONFIG_PATH")
|
||||
configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.")
|
||||
)
|
||||
|
||||
func main() {
|
||||
common.SetupLogging(logDir)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"BIND_ADDRESS": bindAddr,
|
||||
"listen": *bindAddr,
|
||||
"DATABASE": dataSource,
|
||||
"LOG_DIR": logDir,
|
||||
"SERVER_NAME": serverName,
|
||||
"BASE_PATH": basePath,
|
||||
"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
|
||||
"CONFIG_PATH": configPath,
|
||||
"config": *configPath,
|
||||
}).Info("Loading configuration based on config file and environment variables")
|
||||
|
||||
cfg, err := configureServer()
|
||||
@ -64,15 +67,10 @@ func main() {
|
||||
log.WithError(err).Fatal("Invalid configuration")
|
||||
}
|
||||
|
||||
db, err := storage.Open(cfg.DataSource)
|
||||
if err != nil {
|
||||
log.WithError(err).Panic("Failed to open database")
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"BIND_ADDRESS": bindAddr,
|
||||
"listen": *bindAddr,
|
||||
"LOG_DIR": logDir,
|
||||
"CONFIG_PATH": configPath,
|
||||
"CONFIG_PATH": *configPath,
|
||||
"ServerName": cfg.ServerName,
|
||||
"AbsBasePath": cfg.AbsBasePath,
|
||||
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
|
||||
@ -82,13 +80,21 @@ func main() {
|
||||
"ThumbnailSizes": cfg.ThumbnailSizes,
|
||||
}).Info("Starting mediaapi server with configuration")
|
||||
|
||||
db, err := storage.Open(cfg.DataSource)
|
||||
if err != nil {
|
||||
log.WithError(err).Panic("Failed to open database")
|
||||
}
|
||||
|
||||
routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
|
||||
log.Fatal(http.ListenAndServe(bindAddr, nil))
|
||||
log.Fatal(http.ListenAndServe(*bindAddr, nil))
|
||||
}
|
||||
|
||||
// configureServer loads configuration from a yaml file and overrides with environment variables
|
||||
func configureServer() (*config.MediaAPI, error) {
|
||||
cfg, err := loadConfig(configPath)
|
||||
if *configPath == "" {
|
||||
log.Fatal("--config must be supplied")
|
||||
}
|
||||
cfg, err := loadConfig(*configPath)
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("Invalid config file")
|
||||
}
|
||||
@ -172,14 +178,14 @@ func applyOverrides(cfg *config.MediaAPI) {
|
||||
if cfg.MaxThumbnailGenerators == 0 {
|
||||
log.WithField(
|
||||
"max_thumbnail_generators", cfg.MaxThumbnailGenerators,
|
||||
).Info("Using default max_thumbnail_generators")
|
||||
).Info("Using default max_thumbnail_generators value of 10")
|
||||
cfg.MaxThumbnailGenerators = 10
|
||||
}
|
||||
}
|
||||
|
||||
func validateConfig(cfg *config.MediaAPI) error {
|
||||
if bindAddr == "" {
|
||||
return fmt.Errorf("no BIND_ADDRESS environment variable found")
|
||||
if *bindAddr == "" {
|
||||
log.Fatal("--listen must be supplied")
|
||||
}
|
||||
|
||||
absBasePath, err := getAbsolutePath(cfg.BasePath)
|
||||
|
||||
@ -0,0 +1,69 @@
|
||||
# Media API Tests
|
||||
|
||||
## Implemented
|
||||
|
||||
* functional
|
||||
* upload
|
||||
* normal case
|
||||
* download
|
||||
* local file
|
||||
* existing
|
||||
* non-existing
|
||||
* remote file
|
||||
* existing
|
||||
* thumbnail
|
||||
* original file formats
|
||||
* JPEG
|
||||
* local file
|
||||
* existing
|
||||
* remote file
|
||||
* existing
|
||||
* cache
|
||||
* cold
|
||||
* hot
|
||||
* pre-generation according to configuration
|
||||
* scale
|
||||
* crop
|
||||
* dynamic generation
|
||||
* cold cache
|
||||
* larger than original
|
||||
* scale
|
||||
|
||||
## TODO
|
||||
|
||||
* functional
|
||||
* upload
|
||||
* file too large
|
||||
* 0-byte file?
|
||||
* invalid filename
|
||||
* invalid content-type
|
||||
* download
|
||||
* invalid origin
|
||||
* invalid media id
|
||||
* thumbnail
|
||||
* original file formats
|
||||
* GIF
|
||||
* PNG
|
||||
* BMP
|
||||
* SVG
|
||||
* PDF
|
||||
* TIFF
|
||||
* WEBP
|
||||
* local file
|
||||
* non-existing
|
||||
* remote file
|
||||
* non-existing
|
||||
* pre-generation according to configuration
|
||||
* manual verification + hash check for regressions?
|
||||
* dynamic generation
|
||||
* hot cache
|
||||
* limit on dimensions?
|
||||
* 0x0
|
||||
* crop
|
||||
* load
|
||||
* 100 parallel requests
|
||||
* same file
|
||||
* different local files
|
||||
* different remote files
|
||||
* pre-generated thumbnails
|
||||
* non-pre-generated thumbnails
|
||||
@ -0,0 +1,272 @@
|
||||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/matrix-org/dendrite/common/test"
|
||||
)
|
||||
|
||||
var (
|
||||
// How long to wait for the server to write the expected output messages.
|
||||
// This needs to be high enough to account for the time it takes to create
|
||||
// the postgres database tables which can take a while on travis.
|
||||
timeoutString = test.Defaulting(os.Getenv("TIMEOUT"), "10s")
|
||||
// The name of maintenance database to connect to in order to create the test database.
|
||||
postgresDatabase = test.Defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
|
||||
// The name of the test database to create.
|
||||
testDatabaseName = test.Defaulting(os.Getenv("DATABASE_NAME"), "mediaapi_test")
|
||||
// Postgres docker container name (for running psql). If not set, psql must be in PATH.
|
||||
postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
|
||||
// Test image to be uploaded/downloaded
|
||||
testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg")
|
||||
)
|
||||
|
||||
var thumbnailPregenerationConfig = (`
|
||||
thumbnail_sizes:
|
||||
- width: 32
|
||||
height: 32
|
||||
method: crop
|
||||
- width: 96
|
||||
height: 96
|
||||
method: crop
|
||||
- width: 320
|
||||
height: 240
|
||||
method: scale
|
||||
- width: 640
|
||||
height: 480
|
||||
method: scale
|
||||
- width: 800
|
||||
height: 600
|
||||
method: scale
|
||||
`)
|
||||
|
||||
const serverType = "media-api"
|
||||
|
||||
var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes"
|
||||
|
||||
var timeout time.Duration
|
||||
|
||||
func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) {
|
||||
dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
serverAddr := "localhost:177" + suffix + "9"
|
||||
proxyAddr := "localhost:1800" + suffix
|
||||
|
||||
configFilename := serverType + "-server-test-config" + suffix + ".yaml"
|
||||
configFileContents := makeConfig(proxyAddr, suffix, dir, dynamicThumbnails)
|
||||
|
||||
serverArgs := []string{
|
||||
"--config", configFilename,
|
||||
"--listen", serverAddr,
|
||||
}
|
||||
|
||||
databases := []string{
|
||||
testDatabaseName + suffix,
|
||||
}
|
||||
|
||||
proxyCmd, proxyCmdChan := test.StartProxy(
|
||||
proxyAddr,
|
||||
"http://localhost:177"+suffix+"6",
|
||||
"http://localhost:177"+suffix+"8",
|
||||
"http://"+serverAddr,
|
||||
)
|
||||
|
||||
cmd, cmdChan := test.StartServer(
|
||||
serverType,
|
||||
serverArgs,
|
||||
suffix,
|
||||
configFilename,
|
||||
configFileContents,
|
||||
postgresDatabase,
|
||||
postgresContainerName,
|
||||
databases,
|
||||
)
|
||||
|
||||
fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, serverAddr, dir)
|
||||
return cmd, cmdChan, serverAddr, proxyCmd, proxyCmdChan, proxyAddr, dir
|
||||
}
|
||||
|
||||
func makeConfig(serverAddr, suffix, basePath string, dynamicThumbnails bool) string {
|
||||
return fmt.Sprintf(
|
||||
`
|
||||
server_name: "%s"
|
||||
base_path: %s
|
||||
max_file_size_bytes: %s
|
||||
database: "%s"
|
||||
dynamic_thumbnails: %s
|
||||
%s`,
|
||||
serverAddr,
|
||||
basePath,
|
||||
"10485760",
|
||||
fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix),
|
||||
strconv.FormatBool(dynamicThumbnails),
|
||||
thumbnailPregenerationConfig,
|
||||
)
|
||||
}
|
||||
|
||||
func cleanUpServer(cmd *exec.Cmd, dir string) {
|
||||
cmd.Process.Kill() // ensure server is dead, only cleaning up so don't care about errors this returns.
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
fmt.Printf("WARNING: Failed to remove temporary directory %v: %q\n", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Runs a battery of media API server tests
|
||||
// The tests will pause at various points in this list to conduct tests on the HTTP responses before continuing.
|
||||
func main() {
|
||||
fmt.Println("==TESTING==", os.Args[0])
|
||||
|
||||
var err error
|
||||
timeout, err = time.ParseDuration(timeoutString)
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: Invalid timeout string %v: %q\n", timeoutString, err)
|
||||
return
|
||||
}
|
||||
|
||||
// create server1 with only pre-generated thumbnails allowed
|
||||
server1Cmd, server1CmdChan, _, server1ProxyCmd, _, server1ProxyAddr, server1Dir := startMediaAPI("1", false)
|
||||
defer cleanUpServer(server1Cmd, server1Dir)
|
||||
defer server1ProxyCmd.Process.Kill()
|
||||
testDownload(server1ProxyAddr, server1ProxyAddr, "doesnotexist", "", 404, server1CmdChan)
|
||||
|
||||
// upload a JPEG file
|
||||
testUpload(server1ProxyAddr, testJPEG, "image/jpeg", `{
|
||||
"content_uri": "mxc://localhost:18001/1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0"
|
||||
}`, 200, server1CmdChan)
|
||||
|
||||
// download that JPEG file
|
||||
testDownload(server1ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server1CmdChan)
|
||||
|
||||
// thumbnail that JPEG file
|
||||
testThumbnail(64, 64, "crop", server1ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server1CmdChan)
|
||||
|
||||
// create server2 with dynamic thumbnail generation
|
||||
server2Cmd, server2CmdChan, _, server2ProxyCmd, _, server2ProxyAddr, server2Dir := startMediaAPI("2", true)
|
||||
defer cleanUpServer(server2Cmd, server2Dir)
|
||||
defer server2ProxyCmd.Process.Kill()
|
||||
testDownload(server2ProxyAddr, server2ProxyAddr, "doesnotexist", "", 404, server2CmdChan)
|
||||
|
||||
// pre-generated thumbnail that JPEG file via server2
|
||||
testThumbnail(800, 600, "scale", server2ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server2CmdChan)
|
||||
|
||||
// download that JPEG file via server2
|
||||
testDownload(server2ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server2CmdChan)
|
||||
|
||||
// dynamic thumbnail that JPEG file via server2
|
||||
testThumbnail(1920, 1080, "scale", server2ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server2CmdChan)
|
||||
|
||||
// thumbnail that JPEG file via server2
|
||||
testThumbnail(10000, 10000, "scale", server2ProxyAddr, "localhost:18001", "1VuVy8u_hmDllD8BrcY0deM34Bl7SPJeY9J6BkMmpx0", "", 200, server2CmdChan)
|
||||
|
||||
}
|
||||
|
||||
func getMediaURI(scheme, host, endpoint, query string, components []string) string {
|
||||
pathComponents := []string{host, "_matrix/media/v1", endpoint}
|
||||
pathComponents = append(pathComponents, components...)
|
||||
return scheme + path.Join(pathComponents...) + query
|
||||
}
|
||||
|
||||
func testUpload(host, filePath, contentType, wantedBody string, wantedStatusCode int, serverCmdChan chan error) {
|
||||
fmt.Printf("==TESTING== upload %v to %v\n", filePath, host)
|
||||
file, err := os.Open(filePath)
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
filename := filepath.Base(filePath)
|
||||
stat, err := file.Stat()
|
||||
if os.IsNotExist(err) {
|
||||
panic(err)
|
||||
}
|
||||
fileSize := stat.Size()
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"POST",
|
||||
getMediaURI("https://", host, "upload", "?filename="+filename, nil),
|
||||
file,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.ContentLength = fileSize
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
|
||||
testReq := &test.Request{
|
||||
Req: req,
|
||||
WantedStatusCode: wantedStatusCode,
|
||||
WantedBody: test.CanonicalJSONInput([]string{wantedBody})[0],
|
||||
}
|
||||
if err := testReq.Do(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf("==TESTING== upload %v to %v PASSED\n", filePath, host)
|
||||
}
|
||||
|
||||
func testDownload(host, origin, mediaID, wantedBody string, wantedStatusCode int, serverCmdChan chan error) {
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
getMediaURI("https://", host, "download", "", []string{
|
||||
origin,
|
||||
mediaID,
|
||||
}),
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testReq := &test.Request{
|
||||
Req: req,
|
||||
WantedStatusCode: wantedStatusCode,
|
||||
WantedBody: test.CanonicalJSONInput([]string{wantedBody})[0],
|
||||
}
|
||||
testReq.Run(fmt.Sprintf("download mxc://%v/%v from %v", origin, mediaID, host), timeout, serverCmdChan)
|
||||
}
|
||||
|
||||
func testThumbnail(width, height int, resizeMethod, host, origin, mediaID, wantedBody string, wantedStatusCode int, serverCmdChan chan error) {
|
||||
query := fmt.Sprintf("?width=%v&height=%v", width, height)
|
||||
if resizeMethod != "" {
|
||||
query += "&method=" + resizeMethod
|
||||
}
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
getMediaURI("https://", host, "thumbnail", query, []string{
|
||||
origin,
|
||||
mediaID,
|
||||
}),
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testReq := &test.Request{
|
||||
Req: req,
|
||||
WantedStatusCode: wantedStatusCode,
|
||||
WantedBody: test.CanonicalJSONInput([]string{wantedBody})[0],
|
||||
}
|
||||
testReq.Run(fmt.Sprintf("thumbnail mxc://%v/%v%v from %v", origin, mediaID, query, host), timeout, serverCmdChan)
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 MiB |
@ -17,13 +17,10 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/matrix-org/dendrite/common/test"
|
||||
@ -33,23 +30,25 @@ import (
|
||||
|
||||
var (
|
||||
// Path to where kafka is installed.
|
||||
kafkaDir = defaulting(os.Getenv("KAFKA_DIR"), "kafka")
|
||||
kafkaDir = test.Defaulting(os.Getenv("KAFKA_DIR"), "kafka")
|
||||
// The URI the kafka zookeeper is listening on.
|
||||
zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
|
||||
zookeeperURI = test.Defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
|
||||
// The URI the kafka server is listening on.
|
||||
kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
|
||||
kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
|
||||
// The address the syncserver should listen on.
|
||||
syncserverAddr = defaulting(os.Getenv("SYNCSERVER_URI"), "localhost:9876")
|
||||
syncserverAddr = test.Defaulting(os.Getenv("SYNCSERVER_URI"), "localhost:9876")
|
||||
// How long to wait for the syncserver to write the expected output messages.
|
||||
// This needs to be high enough to account for the time it takes to create
|
||||
// the postgres database tables which can take a while on travis.
|
||||
timeoutString = defaulting(os.Getenv("TIMEOUT"), "10s")
|
||||
timeoutString = test.Defaulting(os.Getenv("TIMEOUT"), "10s")
|
||||
// The name of maintenance database to connect to in order to create the test database.
|
||||
postgresDatabase = defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
|
||||
postgresDatabase = test.Defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres")
|
||||
// Postgres docker container name (for running psql). If not set, psql must be in PATH.
|
||||
postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
|
||||
// The name of the test database to create.
|
||||
testDatabaseName = defaulting(os.Getenv("DATABASE_NAME"), "syncserver_test")
|
||||
testDatabaseName = test.Defaulting(os.Getenv("DATABASE_NAME"), "syncserver_test")
|
||||
// The postgres connection config for connecting to the test database.
|
||||
testDatabase = defaulting(os.Getenv("DATABASE"), fmt.Sprintf("dbname=%s sslmode=disable binary_parameters=yes", testDatabaseName))
|
||||
testDatabase = test.Defaulting(os.Getenv("DATABASE"), fmt.Sprintf("dbname=%s sslmode=disable binary_parameters=yes", testDatabaseName))
|
||||
)
|
||||
|
||||
const inputTopic = "syncserverInput"
|
||||
@ -63,36 +62,12 @@ var exe = test.KafkaExecutor{
|
||||
OutputWriter: os.Stderr,
|
||||
}
|
||||
|
||||
var (
|
||||
lastRequestMutex sync.Mutex
|
||||
lastRequestErr error
|
||||
)
|
||||
|
||||
func setLastRequestError(err error) {
|
||||
lastRequestMutex.Lock()
|
||||
defer lastRequestMutex.Unlock()
|
||||
lastRequestErr = err
|
||||
}
|
||||
|
||||
func getLastRequestError() error {
|
||||
lastRequestMutex.Lock()
|
||||
defer lastRequestMutex.Unlock()
|
||||
return lastRequestErr
|
||||
}
|
||||
|
||||
var syncServerConfigFileContents = (`consumer_uris: ["` + kafkaURI + `"]
|
||||
roomserver_topic: "` + inputTopic + `"
|
||||
database: "` + testDatabase + `"
|
||||
server_name: "localhost"
|
||||
`)
|
||||
|
||||
func defaulting(value, defaultValue string) string {
|
||||
if value == "" {
|
||||
value = defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
var timeout time.Duration
|
||||
var clientEventTestData []string
|
||||
|
||||
@ -108,31 +83,6 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: dupes roomserver integration tests. Factor out.
|
||||
func createDatabase(database string) error {
|
||||
cmd := exec.Command("psql", postgresDatabase)
|
||||
cmd.Stdin = strings.NewReader(
|
||||
fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
|
||||
)
|
||||
// Send stdout and stderr to our stderr so that we see error messages from
|
||||
// the psql process
|
||||
cmd.Stdout = os.Stderr
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// TODO: dupes roomserver integration tests. Factor out.
|
||||
func canonicalJSONInput(jsonData []string) []string {
|
||||
for i := range jsonData {
|
||||
jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
jsonData[i] = string(jsonBytes)
|
||||
}
|
||||
return jsonData
|
||||
}
|
||||
|
||||
func createTestUser(database, username, token string) error {
|
||||
cmd := exec.Command(
|
||||
filepath.Join(filepath.Dir(os.Args[0]), "create-account"),
|
||||
@ -172,66 +122,32 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
|
||||
return string(jsonBytes)
|
||||
}
|
||||
|
||||
// doSyncRequest does a /sync request and returns an error if it fails or doesn't
|
||||
// return the wanted string.
|
||||
func doSyncRequest(syncServerURL, want string) error {
|
||||
cli := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
res, err := cli.Get(syncServerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != 200 {
|
||||
return fmt.Errorf("/sync returned HTTP status %d", res.StatusCode)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
resBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(jsonBytes) != want {
|
||||
return fmt.Errorf("/sync returned wrong bytes. Expected:\n%s\n\nGot:\n%s", want, string(jsonBytes))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncRequestUntilSuccess blocks and performs the same /sync request over and over until
|
||||
// the response returns the wanted string, where it will close the given channel and return.
|
||||
// It will keep track of the last error in `lastRequestErr`.
|
||||
func syncRequestUntilSuccess(done chan error, userID, since, want string) {
|
||||
for {
|
||||
sinceQuery := ""
|
||||
if since != "" {
|
||||
sinceQuery = "&since=" + since
|
||||
}
|
||||
err := doSyncRequest(
|
||||
// low value timeout so polling with an up-to-date token returns quickly
|
||||
"http://"+syncserverAddr+"/api/_matrix/client/r0/sync?timeout=100&access_token="+userID+sinceQuery,
|
||||
want,
|
||||
)
|
||||
if err != nil {
|
||||
setLastRequestError(err)
|
||||
time.Sleep(1 * time.Second) // don't tightloop
|
||||
continue
|
||||
}
|
||||
close(done)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// startSyncServer creates the database and config file needed for the sync server to run and
|
||||
// then starts the sync server. The Cmd being executed is returned. A channel is also returned,
|
||||
// which will have any termination errors sent down it, followed immediately by the channel being closed.
|
||||
func startSyncServer() (*exec.Cmd, chan error) {
|
||||
if err := createDatabase(testDatabaseName); err != nil {
|
||||
panic(err)
|
||||
const configFilename = "sync-api-server-config-test.yaml"
|
||||
|
||||
serverArgs := []string{
|
||||
"--config", configFilename,
|
||||
"--listen", syncserverAddr,
|
||||
}
|
||||
|
||||
databases := []string{
|
||||
testDatabaseName,
|
||||
}
|
||||
|
||||
cmd, cmdChan := test.StartServer(
|
||||
"sync-api",
|
||||
serverArgs,
|
||||
"",
|
||||
configFilename,
|
||||
syncServerConfigFileContents,
|
||||
postgresDatabase,
|
||||
postgresContainerName,
|
||||
databases,
|
||||
)
|
||||
|
||||
if err := createTestUser(testDatabase, "alice", "@alice:localhost"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@ -242,29 +158,7 @@ func startSyncServer() (*exec.Cmd, chan error) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
const configFileName = "sync-api-server-config-test.yaml"
|
||||
err := ioutil.WriteFile(configFileName, []byte(syncServerConfigFileContents), 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(
|
||||
filepath.Join(filepath.Dir(os.Args[0]), "dendrite-sync-api-server"),
|
||||
"--config", configFileName,
|
||||
"--listen", syncserverAddr,
|
||||
)
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
panic("failed to start sync server: " + err.Error())
|
||||
}
|
||||
syncServerCmdChan := make(chan error, 1)
|
||||
go func() {
|
||||
syncServerCmdChan <- cmd.Wait()
|
||||
close(syncServerCmdChan)
|
||||
}()
|
||||
return cmd, syncServerCmdChan
|
||||
return cmd, cmdChan
|
||||
}
|
||||
|
||||
// prepareKafka creates the topics which will be written to by the tests.
|
||||
@ -277,49 +171,24 @@ func prepareKafka() {
|
||||
|
||||
func testSyncServer(syncServerCmdChan chan error, userID, since, want string) {
|
||||
fmt.Printf("==TESTING== testSyncServer(%s,%s)\n", userID, since)
|
||||
done := make(chan error, 1)
|
||||
|
||||
// We need to wait for the sync server to:
|
||||
// - have created the tables
|
||||
// - be listening on the given port
|
||||
// - have consumed the kafka logs
|
||||
// before we begin hitting it with /sync requests. We don't get told when it has done
|
||||
// all these things, so we just continually hit /sync until it returns the right bytes.
|
||||
// We can't even wait for the first valid 200 OK response because it's possible to race
|
||||
// with consuming the kafka logs (so the /sync response will be missing events and
|
||||
// therefore fail the test).
|
||||
go syncRequestUntilSuccess(done, userID, since, canonicalJSONInput([]string{want})[0])
|
||||
|
||||
// wait for one of:
|
||||
// - the test to pass (done channel is closed)
|
||||
// - the sync server to exit with an error (error sent on syncServerCmdChan)
|
||||
// - our test timeout to expire
|
||||
// We don't need to clean up since the main() function handles that in the event we panic
|
||||
var testPassed bool
|
||||
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
if testPassed {
|
||||
break
|
||||
}
|
||||
fmt.Printf("==TESTING== testSyncServer(%s,%s) TIMEOUT\n", userID, since)
|
||||
if reqErr := getLastRequestError(); reqErr != nil {
|
||||
fmt.Println("Last /sync request error:")
|
||||
fmt.Println(reqErr)
|
||||
}
|
||||
panic("dendrite-sync-api-server timed out")
|
||||
case err := <-syncServerCmdChan:
|
||||
if err != nil {
|
||||
fmt.Println("=============================================================================================")
|
||||
fmt.Println("sync server failed to run. If failing with 'pq: password authentication failed for user' try:")
|
||||
fmt.Println(" export PGHOST=/var/run/postgresql")
|
||||
fmt.Println("=============================================================================================")
|
||||
panic(err)
|
||||
}
|
||||
case <-done:
|
||||
testPassed = true
|
||||
fmt.Printf("==TESTING== testSyncServer(%s,%s) PASSED\n", userID, since)
|
||||
sinceQuery := ""
|
||||
if since != "" {
|
||||
sinceQuery = "&since=" + since
|
||||
}
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
"http://"+syncserverAddr+"/api/_matrix/client/r0/sync?timeout=100&access_token="+userID+sinceQuery,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
testReq := &test.Request{
|
||||
Req: req,
|
||||
WantedStatusCode: 200,
|
||||
WantedBody: test.CanonicalJSONInput([]string{want})[0],
|
||||
}
|
||||
testReq.Run("sync-api", timeout, syncServerCmdChan)
|
||||
}
|
||||
|
||||
func writeToRoomServerLog(indexes ...int) {
|
||||
@ -327,7 +196,7 @@ func writeToRoomServerLog(indexes ...int) {
|
||||
for _, i := range indexes {
|
||||
roomEvents = append(roomEvents, outputRoomEventTestData[i])
|
||||
}
|
||||
if err := exe.WriteToTopic(inputTopic, canonicalJSONInput(roomEvents)); err != nil {
|
||||
if err := exe.WriteToTopic(inputTopic, test.CanonicalJSONInput(roomEvents)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
163
src/github.com/matrix-org/dendrite/common/test/client.go
Normal file
163
src/github.com/matrix-org/dendrite/common/test/client.go
Normal file
@ -0,0 +1,163 @@
|
||||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package test
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
)
|
||||
|
||||
// Request contains the information necessary to issue a request and test its result
|
||||
type Request struct {
|
||||
Req *http.Request
|
||||
WantedBody string
|
||||
WantedStatusCode int
|
||||
LastErr *LastRequestErr
|
||||
}
|
||||
|
||||
// LastRequestErr is a synchronized error wrapper
|
||||
// Useful for obtaining the last error from a set of requests
|
||||
type LastRequestErr struct {
|
||||
sync.Mutex
|
||||
Err error
|
||||
}
|
||||
|
||||
// Set sets the error
|
||||
func (r *LastRequestErr) Set(err error) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.Err = err
|
||||
}
|
||||
|
||||
// Get gets the error
|
||||
func (r *LastRequestErr) Get() error {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
return r.Err
|
||||
}
|
||||
|
||||
// CanonicalJSONInput canonicalises a slice of JSON strings
|
||||
// Useful for test input
|
||||
func CanonicalJSONInput(jsonData []string) []string {
|
||||
for i := range jsonData {
|
||||
jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
|
||||
if err != nil && err != io.EOF {
|
||||
panic(err)
|
||||
}
|
||||
jsonData[i] = string(jsonBytes)
|
||||
}
|
||||
return jsonData
|
||||
}
|
||||
|
||||
// Do issues a request and checks the status code and body of the response
|
||||
func (r *Request) Do() error {
|
||||
client := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
res, err := client.Do(r.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != r.WantedStatusCode {
|
||||
return fmt.Errorf("incorrect status code. Expected: %d Got: %d", r.WantedStatusCode, res.StatusCode)
|
||||
}
|
||||
|
||||
if r.WantedBody != "" {
|
||||
resBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(jsonBytes) != r.WantedBody {
|
||||
return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
|
||||
// It then closes the given channel and returns.
|
||||
func (r *Request) DoUntilSuccess(done chan error) {
|
||||
r.LastErr = &LastRequestErr{}
|
||||
for {
|
||||
if err := r.Do(); err != nil {
|
||||
r.LastErr.Set(err)
|
||||
time.Sleep(1 * time.Second) // don't tightloop
|
||||
continue
|
||||
}
|
||||
close(done)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Run repeatedly issues a request until success, error or a timeout is reached
|
||||
func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
|
||||
fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
|
||||
done := make(chan error, 1)
|
||||
|
||||
// We need to wait for the server to:
|
||||
// - have connected to the database
|
||||
// - have created the tables
|
||||
// - be listening on the given port
|
||||
go r.DoUntilSuccess(done)
|
||||
|
||||
// wait for one of:
|
||||
// - the test to pass (done channel is closed)
|
||||
// - the server to exit with an error (error sent on serverCmdChan)
|
||||
// - our test timeout to expire
|
||||
// We don't need to clean up since the main() function handles that in the event we panic
|
||||
var testPassed bool
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
if testPassed {
|
||||
break
|
||||
}
|
||||
fmt.Printf("==TESTING== %v TIMEOUT\n", label)
|
||||
if reqErr := r.LastErr.Get(); reqErr != nil {
|
||||
fmt.Println("Last /sync request error:")
|
||||
fmt.Println(reqErr)
|
||||
}
|
||||
panic(fmt.Sprintf("%v server timed out", label))
|
||||
case err := <-serverCmdChan:
|
||||
if err != nil {
|
||||
fmt.Println("=============================================================================================")
|
||||
fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
|
||||
fmt.Println(" export PGHOST=/var/run/postgresql")
|
||||
fmt.Println("=============================================================================================")
|
||||
panic(err)
|
||||
}
|
||||
case <-done:
|
||||
testPassed = true
|
||||
fmt.Printf("==TESTING== %v PASSED\n", label)
|
||||
}
|
||||
}
|
||||
116
src/github.com/matrix-org/dendrite/common/test/server.go
Normal file
116
src/github.com/matrix-org/dendrite/common/test/server.go
Normal file
@ -0,0 +1,116 @@
|
||||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Defaulting allows assignment of string variables with a fallback default value
|
||||
// Useful for use with os.Getenv() for example
|
||||
func Defaulting(value, defaultValue string) string {
|
||||
if value == "" {
|
||||
value = defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// CreateDatabase creates a new database, dropping it first if it exists
|
||||
func CreateDatabase(command string, args []string, database string) error {
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Stdin = strings.NewReader(
|
||||
fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
|
||||
)
|
||||
// Send stdout and stderr to our stderr so that we see error messages from
|
||||
// the psql process
|
||||
cmd.Stdout = os.Stderr
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// CreateBackgroundCommand creates an executable command
|
||||
// The Cmd being executed is returned. A channel is also returned,
|
||||
// which will have any termination errors sent down it, followed immediately by the channel being closed.
|
||||
func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
panic("failed to start server: " + err.Error())
|
||||
}
|
||||
cmdChan := make(chan error, 1)
|
||||
go func() {
|
||||
cmdChan <- cmd.Wait()
|
||||
close(cmdChan)
|
||||
}()
|
||||
return cmd, cmdChan
|
||||
}
|
||||
|
||||
// StartServer creates the database and config file needed for the server to run and
|
||||
// then starts the server. The Cmd being executed is returned. A channel is also returned,
|
||||
// which will have any termination errors sent down it, followed immediately by the channel being closed.
|
||||
// If postgresContainerName is not an empty string, psql will be run from inside that container. If it is
|
||||
// an empty string, psql will be assumed to be in PATH.
|
||||
func StartServer(serverType string, serverArgs []string, suffix, configFilename, configFileContents, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) {
|
||||
if len(databases) > 0 {
|
||||
var dbCmd string
|
||||
var dbArgs []string
|
||||
if postgresContainerName == "" {
|
||||
dbCmd = "psql"
|
||||
dbArgs = []string{postgresDatabase}
|
||||
} else {
|
||||
dbCmd = "docker"
|
||||
dbArgs = []string{
|
||||
"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
|
||||
}
|
||||
}
|
||||
for _, database := range databases {
|
||||
if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if configFilename != "" {
|
||||
if err := ioutil.WriteFile(configFilename, []byte(configFileContents), 0644); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
return CreateBackgroundCommand(
|
||||
filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"),
|
||||
serverArgs,
|
||||
)
|
||||
}
|
||||
|
||||
// StartProxy creates a reverse proxy
|
||||
func StartProxy(bindAddr, syncAddr, clientAddr, mediaAddr string) (*exec.Cmd, chan error) {
|
||||
proxyArgs := []string{
|
||||
"--bind-address", bindAddr,
|
||||
"--sync-api-server-url", syncAddr,
|
||||
"--client-api-server-url", clientAddr,
|
||||
"--media-api-server-url", mediaAddr,
|
||||
}
|
||||
return CreateBackgroundCommand(
|
||||
filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
|
||||
proxyArgs,
|
||||
)
|
||||
}
|
||||
@ -17,10 +17,12 @@ package thumbnailer
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
)
|
||||
|
||||
@ -131,6 +133,24 @@ func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.Active
|
||||
delete(activeThumbnailGeneration.PathToResult, string(dst))
|
||||
}
|
||||
|
||||
func isThumbnailExists(dst types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, db *storage.Database, logger *log.Entry) (bool, error) {
|
||||
thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
|
||||
if err != nil {
|
||||
logger.Error("Failed to query database for thumbnail.")
|
||||
return false, err
|
||||
}
|
||||
if thumbnailMetadata != nil {
|
||||
return true, nil
|
||||
}
|
||||
// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
|
||||
// The functions are error checkers to be used in different cases.
|
||||
if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
|
||||
// Thumbnail exists
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// init with worst values
|
||||
func newThumbnailFitness() thumbnailFitness {
|
||||
return thumbnailFitness{
|
||||
|
||||
@ -17,7 +17,6 @@
|
||||
package thumbnailer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
@ -34,9 +33,10 @@ func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMeta
|
||||
logger.WithError(err).WithField("src", src).Error("Failed to read src file")
|
||||
return false, err
|
||||
}
|
||||
img := bimg.NewImage(buffer)
|
||||
for _, config := range configs {
|
||||
// Note: createThumbnail does locking based on activeThumbnailGeneration
|
||||
busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
if err != nil {
|
||||
logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
|
||||
return false, err
|
||||
@ -57,8 +57,9 @@ func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata
|
||||
}).Error("Failed to read src file")
|
||||
return false, err
|
||||
}
|
||||
img := bimg.NewImage(buffer)
|
||||
// Note: createThumbnail does locking based on activeThumbnailGeneration
|
||||
busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
if err != nil {
|
||||
logger.WithError(err).WithFields(log.Fields{
|
||||
"src": src,
|
||||
@ -73,13 +74,18 @@ func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata
|
||||
|
||||
// createThumbnail checks if the thumbnail exists, and if not, generates it
|
||||
// Thumbnail generation is only done once for each non-existing thumbnail.
|
||||
func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
func createThumbnail(src types.Path, img *bimg.Image, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
logger = logger.WithFields(log.Fields{
|
||||
"Width": config.Width,
|
||||
"Height": config.Height,
|
||||
"ResizeMethod": config.ResizeMethod,
|
||||
})
|
||||
|
||||
// Check if request is larger than original
|
||||
if isLargerThanOriginal(config, img) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
dst := GetThumbnailPath(src, config)
|
||||
|
||||
// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
|
||||
@ -104,30 +110,13 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
|
||||
}()
|
||||
}
|
||||
|
||||
// Check if the thumbnail exists.
|
||||
thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
|
||||
if err != nil {
|
||||
logger.Error("Failed to query database for thumbnail.")
|
||||
exists, err := isThumbnailExists(dst, config, mediaMetadata, db, logger)
|
||||
if err != nil || exists {
|
||||
return false, err
|
||||
}
|
||||
if thumbnailMetadata != nil {
|
||||
return false, nil
|
||||
}
|
||||
// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
|
||||
// The functions are error checkers to be used in different cases.
|
||||
if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
|
||||
// Thumbnail exists
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if isActive == false {
|
||||
// Note: This should not happen, but we check just in case.
|
||||
logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.")
|
||||
return false, fmt.Errorf("Not active thumbnail generator. Stat error: %q", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
width, height, err := resize(dst, buffer, config.Width, config.Height, config.ResizeMethod == "crop", logger)
|
||||
width, height, err := resize(dst, img, config.Width, config.Height, config.ResizeMethod == "crop", logger)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@ -142,7 +131,7 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
|
||||
return false, err
|
||||
}
|
||||
|
||||
thumbnailMetadata = &types.ThumbnailMetadata{
|
||||
thumbnailMetadata := &types.ThumbnailMetadata{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
MediaID: mediaMetadata.MediaID,
|
||||
Origin: mediaMetadata.Origin,
|
||||
@ -151,8 +140,8 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
|
||||
FileSizeBytes: types.FileSizeBytes(stat.Size()),
|
||||
},
|
||||
ThumbnailSize: types.ThumbnailSize{
|
||||
Width: width,
|
||||
Height: height,
|
||||
Width: config.Width,
|
||||
Height: config.Height,
|
||||
ResizeMethod: config.ResizeMethod,
|
||||
},
|
||||
}
|
||||
@ -169,12 +158,18 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func isLargerThanOriginal(config types.ThumbnailSize, img *bimg.Image) bool {
|
||||
imgSize, err := img.Size()
|
||||
if err == nil && config.Width >= imgSize.Width && config.Height >= imgSize.Height {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// resize scales an image to fit within the provided width and height
|
||||
// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested
|
||||
// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off
|
||||
func resize(dst types.Path, buffer []byte, w, h int, crop bool, logger *log.Entry) (int, int, error) {
|
||||
inImage := bimg.NewImage(buffer)
|
||||
|
||||
func resize(dst types.Path, inImage *bimg.Image, w, h int, crop bool, logger *log.Entry) (int, int, error) {
|
||||
inSize, err := inImage.Size()
|
||||
if err != nil {
|
||||
return -1, -1, err
|
||||
|
||||
@ -17,7 +17,6 @@
|
||||
package thumbnailer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
// Imported for gif codec
|
||||
@ -114,6 +113,11 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
|
||||
"ResizeMethod": config.ResizeMethod,
|
||||
})
|
||||
|
||||
// Check if request is larger than original
|
||||
if config.Width >= img.Bounds().Dx() && config.Height >= img.Bounds().Dy() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
dst := GetThumbnailPath(src, config)
|
||||
|
||||
// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
|
||||
@ -138,27 +142,10 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
|
||||
}()
|
||||
}
|
||||
|
||||
// Check if the thumbnail exists.
|
||||
thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
|
||||
if err != nil {
|
||||
logger.Error("Failed to query database for thumbnail.")
|
||||
exists, err := isThumbnailExists(dst, config, mediaMetadata, db, logger)
|
||||
if err != nil || exists {
|
||||
return false, err
|
||||
}
|
||||
if thumbnailMetadata != nil {
|
||||
return false, nil
|
||||
}
|
||||
// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
|
||||
// The functions are error checkers to be used in different cases.
|
||||
if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
|
||||
// Thumbnail exists
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if isActive == false {
|
||||
// Note: This should not happen, but we check just in case.
|
||||
logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.")
|
||||
return false, fmt.Errorf("Not active thumbnail generator. Stat error: %q", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
width, height, err := adjustSize(dst, img, config.Width, config.Height, config.ResizeMethod == "crop", logger)
|
||||
@ -176,7 +163,7 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
|
||||
return false, err
|
||||
}
|
||||
|
||||
thumbnailMetadata = &types.ThumbnailMetadata{
|
||||
thumbnailMetadata := &types.ThumbnailMetadata{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
MediaID: mediaMetadata.MediaID,
|
||||
Origin: mediaMetadata.Origin,
|
||||
@ -185,8 +172,8 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
|
||||
FileSizeBytes: types.FileSizeBytes(stat.Size()),
|
||||
},
|
||||
ThumbnailSize: types.ThumbnailSize{
|
||||
Width: width,
|
||||
Height: height,
|
||||
Width: config.Width,
|
||||
Height: config.Height,
|
||||
ResizeMethod: config.ResizeMethod,
|
||||
},
|
||||
}
|
||||
|
||||
@ -8,6 +8,9 @@ gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests
|
||||
gb build github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server
|
||||
gb build github.com/matrix-org/dendrite/cmd/syncserver-integration-tests
|
||||
gb build github.com/matrix-org/dendrite/cmd/create-account
|
||||
gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server
|
||||
gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests
|
||||
gb build github.com/matrix-org/dendrite/cmd/client-api-proxy
|
||||
|
||||
# Run the pre commit hooks
|
||||
./hooks/pre-commit
|
||||
@ -15,3 +18,4 @@ gb build github.com/matrix-org/dendrite/cmd/create-account
|
||||
# Run the integration tests
|
||||
bin/roomserver-integration-tests
|
||||
bin/syncserver-integration-tests
|
||||
bin/mediaapi-integration-tests
|
||||
|
||||
Loading…
Reference in New Issue
Block a user