Add Cloudron packaging for Maubot

maubot-src/maubot/__init__.py

@@ -0,0 +1,4 @@
+from .__meta__ import __version__
+from .matrix import MaubotMatrixClient as Client, MaubotMessageEvent as MessageEvent
+from .plugin_base import Plugin
+from .plugin_server import PluginWebApp

maubot-src/maubot/__main__.py

@@ -0,0 +1,185 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import asyncio
+import sys
+
+from mautrix.api import HTTPAPI
+from mautrix.util.async_db import Database, DatabaseException, PostgresDatabase, Scheme
+from mautrix.util.program import Program
+
+from .__meta__ import __version__
+from .client import Client
+from .config import Config
+from .db import init as init_db, upgrade_table
+from .instance import PluginInstance
+from .lib.future_awaitable import FutureAwaitable
+from .lib.state_store import PgStateStore
+from .loader.zip import init as init_zip_loader
+from .management.api import init as init_mgmt_api
+from .server import MaubotServer
+
+try:
+    from mautrix.crypto.store import PgCryptoStore
+except ImportError:
+    PgCryptoStore = None
+
+
+class Maubot(Program):
+    config: Config
+    server: MaubotServer
+    db: Database
+    crypto_db: Database | None
+    plugin_postgres_db: PostgresDatabase | None
+    state_store: PgStateStore
+
+    config_class = Config
+    module = "maubot"
+    name = "maubot"
+    version = __version__
+    command = "python -m maubot"
+    description = "A plugin-based Matrix bot system."
+
+    def prepare_log_websocket(self) -> None:
+        from .management.api.log import init, stop_all
+
+        init(self.loop)
+        self.add_shutdown_actions(FutureAwaitable(stop_all))
+
+    def prepare_arg_parser(self) -> None:
+        super().prepare_arg_parser()
+        self.parser.add_argument(
+            "--ignore-unsupported-database",
+            action="store_true",
+            help="Run even if the database schema is too new",
+        )
+        self.parser.add_argument(
+            "--ignore-foreign-tables",
+            action="store_true",
+            help="Run even if the database contains tables from other programs (like Synapse)",
+        )
+
+    def prepare_db(self) -> None:
+        self.db = Database.create(
+            self.config["database"],
+            upgrade_table=upgrade_table,
+            db_args=self.config["database_opts"],
+            owner_name=self.name,
+            ignore_foreign_tables=self.args.ignore_foreign_tables,
+        )
+        init_db(self.db)
+
+        if PgCryptoStore:
+            if self.config["crypto_database"] == "default":
+                self.crypto_db = self.db
+            else:
+                self.crypto_db = Database.create(
+                    self.config["crypto_database"],
+                    upgrade_table=PgCryptoStore.upgrade_table,
+                    ignore_foreign_tables=self.args.ignore_foreign_tables,
+                )
+        else:
+            self.crypto_db = None
+
+        if self.config["plugin_databases.postgres"] == "default":
+            if self.db.scheme != Scheme.POSTGRES:
+                self.log.critical(
+                    'Using "default" as the postgres plugin database URL is only allowed if '
+                    "the default database is postgres."
+                )
+                sys.exit(24)
+            assert isinstance(self.db, PostgresDatabase)
+            self.plugin_postgres_db = self.db
+        elif self.config["plugin_databases.postgres"]:
+            plugin_db = Database.create(
+                self.config["plugin_databases.postgres"],
+                db_args={
+                    **self.config["database_opts"],
+                    **self.config["plugin_databases.postgres_opts"],
+                },
+            )
+            if plugin_db.scheme != Scheme.POSTGRES:
+                self.log.critical("The plugin postgres database URL must be a postgres database")
+                sys.exit(24)
+            assert isinstance(plugin_db, PostgresDatabase)
+            self.plugin_postgres_db = plugin_db
+        else:
+            self.plugin_postgres_db = None
+
+    def prepare(self) -> None:
+        super().prepare()
+
+        if self.config["api_features.log"]:
+            self.prepare_log_websocket()
+
+        HTTPAPI.default_ua = f"maubot/{self.version} {HTTPAPI.default_ua}"
+        init_zip_loader(self.config)
+        self.prepare_db()
+        Client.init_cls(self)
+        PluginInstance.init_cls(self)
+        management_api = init_mgmt_api(self.config, self.loop)
+        self.server = MaubotServer(management_api, self.config, self.loop)
+        self.state_store = PgStateStore(self.db)
+
+    async def start_db(self) -> None:
+        self.log.debug("Starting database...")
+        ignore_unsupported = self.args.ignore_unsupported_database
+        self.db.upgrade_table.allow_unsupported = ignore_unsupported
+        self.state_store.upgrade_table.allow_unsupported = ignore_unsupported
+        try:
+            await self.db.start()
+            await self.state_store.upgrade_table.upgrade(self.db)
+            if self.plugin_postgres_db and self.plugin_postgres_db is not self.db:
+                await self.plugin_postgres_db.start()
+            if self.crypto_db:
+                PgCryptoStore.upgrade_table.allow_unsupported = ignore_unsupported
+                if self.crypto_db is not self.db:
+                    await self.crypto_db.start()
+                else:
+                    await PgCryptoStore.upgrade_table.upgrade(self.db)
+        except DatabaseException as e:
+            self.log.critical("Failed to initialize database", exc_info=e)
+            if e.explanation:
+                self.log.info(e.explanation)
+            sys.exit(25)
+
+    async def system_exit(self) -> None:
+        if hasattr(self, "db"):
+            self.log.trace("Stopping database due to SystemExit")
+            await self.db.stop()
+
+    async def start(self) -> None:
+        await self.start_db()
+        await asyncio.gather(*[plugin.load() async for plugin in PluginInstance.all()])
+        await asyncio.gather(*[client.start() async for client in Client.all()])
+        await super().start()
+        async for plugin in PluginInstance.all():
+            await plugin.load()
+        await self.server.start()
+
+    async def stop(self) -> None:
+        self.add_shutdown_actions(*(client.stop() for client in Client.cache.values()))
+        await super().stop()
+        self.log.debug("Stopping server")
+        try:
+            await asyncio.wait_for(self.server.stop(), 5)
+        except asyncio.TimeoutError:
+            self.log.warning("Stopping server timed out")
+        await self.db.stop()
+
+
+Maubot().run()

maubot-src/maubot/__meta__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0b1"

maubot-src/maubot/cli/__init__.py

@@ -0,0 +1,2 @@
+from . import commands
+from .base import app

maubot-src/maubot/cli/__main__.py

@@ -0,0 +1,3 @@
+from . import app
+
+app()

maubot-src/maubot/cli/base.py

@@ -0,0 +1,23 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import click
+
+from .config import load_config
+
+
+@click.group()
+def app() -> None:
+    load_config()

maubot-src/maubot/cli/cliq/__init__.py

@@ -0,0 +1,2 @@
+from .cliq import command, option
+from .validators import PathValidator, SPDXValidator, VersionValidator

maubot-src/maubot/cli/cliq/cliq.py

@@ -0,0 +1,169 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, Callable
+import asyncio
+import functools
+import inspect
+import traceback
+
+from colorama import Fore
+from prompt_toolkit.validation import Validator
+from questionary import prompt
+import aiohttp
+import click
+
+from ..base import app
+from ..config import get_token
+from .validators import ClickValidator, Required
+
+
+def with_http(func):
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs):
+        async with aiohttp.ClientSession() as sess:
+            try:
+                return await func(*args, sess=sess, **kwargs)
+            except aiohttp.ClientError as e:
+                print(f"{Fore.RED}Connection error: {e}{Fore.RESET}")
+
+    return wrapper
+
+
+def with_authenticated_http(func):
+    @functools.wraps(func)
+    async def wrapper(*args, server: str, **kwargs):
+        server, token = get_token(server)
+        if not token:
+            return
+        async with aiohttp.ClientSession(headers={"Authorization": f"Bearer {token}"}) as sess:
+            try:
+                return await func(*args, sess=sess, server=server, **kwargs)
+            except aiohttp.ClientError as e:
+                print(f"{Fore.RED}Connection error: {e}{Fore.RESET}")
+
+    return wrapper
+
+
+def command(help: str) -> Callable[[Callable], Callable]:
+    def decorator(func) -> Callable:
+        questions = getattr(func, "__inquirer_questions__", {}).copy()
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            for key, value in kwargs.items():
+                if key not in questions:
+                    continue
+                if value is not None and (questions[key]["type"] != "confirm" or value != "null"):
+                    questions.pop(key, None)
+                try:
+                    required_unless = questions[key].pop("required_unless")
+                    if isinstance(required_unless, str) and kwargs[required_unless]:
+                        questions.pop(key)
+                    elif isinstance(required_unless, list):
+                        for v in required_unless:
+                            if kwargs[v]:
+                                questions.pop(key)
+                                break
+                    elif isinstance(required_unless, dict):
+                        for k, v in required_unless.items():
+                            if kwargs.get(v, object()) == v:
+                                questions.pop(key)
+                                break
+                except KeyError:
+                    pass
+            question_list = list(questions.values())
+            question_list.reverse()
+            resp = prompt(question_list, kbi_msg="Aborted!")
+            if not resp and question_list:
+                return
+            kwargs = {**kwargs, **resp}
+
+            try:
+                res = func(*args, **kwargs)
+                if inspect.isawaitable(res):
+                    asyncio.run(res)
+            except Exception:
+                print(Fore.RED + "Fatal error running command" + Fore.RESET)
+                traceback.print_exc()
+
+        return app.command(help=help)(wrapper)
+
+    return decorator
+
+
+def yesno(val: str) -> bool | None:
+    if not val:
+        return None
+    elif isinstance(val, bool):
+        return val
+    elif val.lower() in ("true", "t", "yes", "y"):
+        return True
+    elif val.lower() in ("false", "f", "no", "n"):
+        return False
+
+
+yesno.__name__ = "yes/no"
+
+
+def option(
+    short: str,
+    long: str,
+    message: str = None,
+    help: str = None,
+    click_type: str | Callable[[str], Any] = None,
+    inq_type: str = None,
+    validator: type[Validator] = None,
+    required: bool = False,
+    default: str | bool | None = None,
+    is_flag: bool = False,
+    prompt: bool = True,
+    required_unless: str | list | dict = None,
+) -> Callable[[Callable], Callable]:
+    if not message:
+        message = long[2].upper() + long[3:]
+
+    if isinstance(validator, type) and issubclass(validator, ClickValidator):
+        click_type = validator.click_type
+    if is_flag:
+        click_type = yesno
+
+    def decorator(func) -> Callable:
+        click.option(short, long, help=help, type=click_type)(func)
+        if not prompt:
+            return func
+        if not hasattr(func, "__inquirer_questions__"):
+            func.__inquirer_questions__ = {}
+        q = {
+            "type": (
+                inq_type if isinstance(inq_type, str) else ("input" if not is_flag else "confirm")
+            ),
+            "name": long[2:],
+            "message": message,
+        }
+        if required_unless is not None:
+            q["required_unless"] = required_unless
+        if default is not None:
+            q["default"] = default
+        if required or required_unless is not None:
+            q["validate"] = Required(validator)
+        elif validator:
+            q["validate"] = validator
+        func.__inquirer_questions__[long[2:]] = q
+        return func
+
+    return decorator

maubot-src/maubot/cli/cliq/validators.py

@@ -0,0 +1,85 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Callable
+import os
+
+from packaging.version import InvalidVersion, Version
+from prompt_toolkit.document import Document
+from prompt_toolkit.validation import ValidationError, Validator
+import click
+
+from ..util import spdx as spdxlib
+
+
+class Required(Validator):
+    proxy: Validator
+
+    def __init__(self, proxy: Validator = None) -> None:
+        self.proxy = proxy
+
+    def validate(self, document: Document) -> None:
+        if len(document.text) == 0:
+            raise ValidationError(message="This field is required")
+        if self.proxy:
+            return self.proxy.validate(document)
+
+
+class ClickValidator(Validator):
+    click_type: Callable[[str], str] = None
+
+    @classmethod
+    def validate(cls, document: Document) -> None:
+        try:
+            cls.click_type(document.text)
+        except click.BadParameter as e:
+            raise ValidationError(message=e.message, cursor_position=len(document.text))
+
+
+def path(val: str) -> str:
+    val = os.path.abspath(val)
+    if os.path.exists(val):
+        return val
+    directory = os.path.dirname(val)
+    if not os.path.isdir(directory):
+        if os.path.exists(directory):
+            raise click.BadParameter(f"{directory} is not a directory")
+        raise click.BadParameter(f"{directory} does not exist")
+    return val
+
+
+class PathValidator(ClickValidator):
+    click_type = path
+
+
+def version(val: str) -> Version:
+    try:
+        return Version(val)
+    except InvalidVersion as e:
+        raise click.BadParameter(f"{val} is not a valid PEP-440 version") from e
+
+
+class VersionValidator(ClickValidator):
+    click_type = version
+
+
+def spdx(val: str) -> str:
+    if not spdxlib.valid(val):
+        raise click.BadParameter(f"{val} is not a valid SPDX license identifier")
+    return val
+
+
+class SPDXValidator(ClickValidator):
+    click_type = spdx

maubot-src/maubot/cli/commands/__init__.py

@@ -0,0 +1 @@
+from . import auth, build, init, login, logs, upload

maubot-src/maubot/cli/commands/auth.py

@@ -0,0 +1,166 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import json
+import webbrowser
+
+from colorama import Fore
+from yarl import URL
+import aiohttp
+import click
+
+from ..cliq import cliq
+
+history_count: int = 10
+
+friendly_errors = {
+    "server_not_found": (
+        "Registration target server not found.\n\n"
+        "To log in or register through maubot, you must add the server to the\n"
+        "homeservers section in the config. If you only want to log in,\n"
+        "leave the `secret` field empty."
+    ),
+    "registration_no_sso": (
+        "The register operation is only for registering with a password.\n\n"
+        "To register with SSO, simply leave out the --register flag."
+    ),
+}
+
+
+async def list_servers(server: str, sess: aiohttp.ClientSession) -> None:
+    url = URL(server) / "_matrix/maubot/v1/client/auth/servers"
+    async with sess.get(url) as resp:
+        data = await resp.json()
+        print(f"{Fore.GREEN}Available Matrix servers for registration and login:{Fore.RESET}")
+        for server in data.keys():
+            print(f"* {Fore.CYAN}{server}{Fore.RESET}")
+
+
+@cliq.command(help="Log into a Matrix account via the Maubot server")
+@cliq.option("-h", "--homeserver", help="The homeserver to log into", required_unless="list")
+@cliq.option(
+    "-u", "--username", help="The username to log in with", required_unless=["list", "sso"]
+)
+@cliq.option(
+    "-p",
+    "--password",
+    help="The password to log in with",
+    inq_type="password",
+    required_unless=["list", "sso"],
+)
+@cliq.option(
+    "-s",
+    "--server",
+    help="The maubot instance to log in through",
+    default="",
+    required=False,
+    prompt=False,
+)
+@click.option(
+    "-r", "--register", help="Register instead of logging in", is_flag=True, default=False
+)
+@click.option(
+    "-c",
+    "--update-client",
+    help="Instead of returning the access token, create or update a client in maubot using it",
+    is_flag=True,
+    default=False,
+)
+@click.option("-l", "--list", help="List available homeservers", is_flag=True, default=False)
+@click.option(
+    "-o", "--sso", help="Use single sign-on instead of password login", is_flag=True, default=False
+)
+@click.option(
+    "-n",
+    "--device-name",
+    help="The initial e2ee device displayname (only for login)",
+    default="Maubot",
+    required=False,
+)
+@cliq.with_authenticated_http
+async def auth(
+    homeserver: str,
+    username: str,
+    password: str,
+    server: str,
+    register: bool,
+    list: bool,
+    update_client: bool,
+    device_name: str,
+    sso: bool,
+    sess: aiohttp.ClientSession,
+) -> None:
+    if list:
+        await list_servers(server, sess)
+        return
+    endpoint = "register" if register else "login"
+    url = URL(server) / "_matrix/maubot/v1/client/auth" / homeserver / endpoint
+    if update_client:
+        url = url.update_query({"update_client": "true"})
+    if sso:
+        url = url.update_query({"sso": "true"})
+        req_data = {"device_name": device_name}
+    else:
+        req_data = {"username": username, "password": password, "device_name": device_name}
+
+    async with sess.post(url, json=req_data) as resp:
+        if not 200 <= resp.status < 300:
+            await print_error(resp, is_register=register)
+        elif sso:
+            await wait_sso(resp, sess, server, homeserver)
+        else:
+            await print_response(resp, is_register=register)
+
+
+async def wait_sso(
+    resp: aiohttp.ClientResponse, sess: aiohttp.ClientSession, server: str, homeserver: str
+) -> None:
+    data = await resp.json()
+    sso_url, reg_id = data["sso_url"], data["id"]
+    print(f"{Fore.GREEN}Opening {Fore.CYAN}{sso_url}{Fore.RESET}")
+    webbrowser.open(sso_url, autoraise=True)
+    print(f"{Fore.GREEN}Waiting for login token...{Fore.RESET}")
+    wait_url = URL(server) / "_matrix/maubot/v1/client/auth" / homeserver / "sso" / reg_id / "wait"
+    async with sess.post(wait_url, json={}) as resp:
+        await print_response(resp, is_register=False)
+
+
+async def print_response(resp: aiohttp.ClientResponse, is_register: bool) -> None:
+    if resp.status == 200:
+        data = await resp.json()
+        action = "registered" if is_register else "logged in as"
+        print(f"{Fore.GREEN}Successfully {action} {Fore.CYAN}{data['user_id']}{Fore.GREEN}.")
+        print(f"{Fore.GREEN}Access token: {Fore.CYAN}{data['access_token']}{Fore.RESET}")
+        print(f"{Fore.GREEN}Device ID: {Fore.CYAN}{data['device_id']}{Fore.RESET}")
+    elif resp.status in (201, 202):
+        data = await resp.json()
+        action = "created" if resp.status == 201 else "updated"
+        print(
+            f"{Fore.GREEN}Successfully {action} client for "
+            f"{Fore.CYAN}{data['id']}{Fore.GREEN} / "
+            f"{Fore.CYAN}{data['device_id']}{Fore.GREEN}.{Fore.RESET}"
+        )
+    else:
+        await print_error(resp, is_register)
+
+
+async def print_error(resp: aiohttp.ClientResponse, is_register: bool) -> None:
+    try:
+        err_data = await resp.json()
+        error = friendly_errors.get(err_data["errcode"], err_data["error"])
+    except (aiohttp.ContentTypeError, json.JSONDecodeError, KeyError):
+        error = await resp.text()
+    action = "register" if is_register else "log in"
+    print(f"{Fore.RED}Failed to {action}: {error}{Fore.RESET}")

maubot-src/maubot/cli/commands/build.py

@@ -0,0 +1,155 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import IO
+from io import BytesIO
+import asyncio
+import glob
+import os
+import zipfile
+
+from aiohttp import ClientSession
+from colorama import Fore
+from questionary import prompt
+from ruamel.yaml import YAML, YAMLError
+import click
+
+from mautrix.types import SerializerError
+
+from ...loader import PluginMeta
+from ..base import app
+from ..cliq import cliq
+from ..cliq.validators import PathValidator
+from ..config import get_token
+from .upload import upload_file
+
+yaml = YAML()
+
+
+def zipdir(zip, dir):
+    for root, dirs, files in os.walk(dir):
+        for file in files:
+            zip.write(os.path.join(root, file))
+
+
+def read_meta(path: str) -> PluginMeta | None:
+    try:
+        with open(os.path.join(path, "maubot.yaml")) as meta_file:
+            try:
+                meta_dict = yaml.load(meta_file)
+            except YAMLError as e:
+                print(Fore.RED + "Failed to build plugin: Metadata file is not YAML")
+                print(Fore.RED + str(e) + Fore.RESET)
+                return None
+    except FileNotFoundError:
+        print(Fore.RED + "Failed to build plugin: Metadata file not found" + Fore.RESET)
+        return None
+    try:
+        meta = PluginMeta.deserialize(meta_dict)
+    except SerializerError as e:
+        print(Fore.RED + "Failed to build plugin: Metadata file is not valid")
+        print(Fore.RED + str(e) + Fore.RESET)
+        return None
+    return meta
+
+
+def read_output_path(output: str, meta: PluginMeta) -> str | None:
+    directory = os.getcwd()
+    filename = f"{meta.id}-v{meta.version}.mbp"
+    if not output:
+        output = os.path.join(directory, filename)
+    elif os.path.isdir(output):
+        output = os.path.join(output, filename)
+    elif os.path.exists(output):
+        q = [{"type": "confirm", "name": "override", "message": f"{output} exists, override?"}]
+        override = prompt(q)["override"]
+        if not override:
+            return None
+        os.remove(output)
+    return os.path.abspath(output)
+
+
+def write_plugin(meta: PluginMeta, output: str | IO) -> None:
+    with zipfile.ZipFile(output, "w") as zip:
+        meta_dump = BytesIO()
+        yaml.dump(meta.serialize(), meta_dump)
+        zip.writestr("maubot.yaml", meta_dump.getvalue())
+
+        for module in meta.modules:
+            if os.path.isfile(f"{module}.py"):
+                zip.write(f"{module}.py")
+            elif module is not None and os.path.isdir(module):
+                if os.path.isfile(f"{module}/__init__.py"):
+                    zipdir(zip, module)
+                else:
+                    print(
+                        Fore.YELLOW
+                        + f"Module {module} is missing __init__.py, skipping"
+                        + Fore.RESET
+                    )
+            else:
+                print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET)
+        for pattern in meta.extra_files:
+            for file in glob.iglob(pattern):
+                zip.write(file)
+
+
+@cliq.with_authenticated_http
+async def upload_plugin(output: str | IO, *, server: str, sess: ClientSession) -> None:
+    server, token = get_token(server)
+    if not token:
+        return
+    if isinstance(output, str):
+        with open(output, "rb") as file:
+            await upload_file(sess, file, server)
+    else:
+        await upload_file(sess, output, server)
+
+
+@app.command(
+    short_help="Build a maubot plugin",
+    help=(
+        "Build a maubot plugin. First parameter is the path to root of the plugin "
+        "to build. You can also use --output to specify output file."
+    ),
+)
+@click.argument("path", default=os.getcwd())
+@click.option(
+    "-o", "--output", help="Path to output built plugin to", type=PathValidator.click_type
+)
+@click.option(
+    "-u", "--upload", help="Upload plugin to server after building", is_flag=True, default=False
+)
+@click.option("-s", "--server", help="Server to upload built plugin to")
+def build(path: str, output: str, upload: bool, server: str) -> None:
+    meta = read_meta(path)
+    if not meta:
+        return
+    if output or not upload:
+        output = read_output_path(output, meta)
+        if not output:
+            return
+    else:
+        output = BytesIO()
+    os.chdir(path)
+    write_plugin(meta, output)
+    if isinstance(output, str):
+        print(f"{Fore.GREEN}Plugin built to {Fore.CYAN}{output}{Fore.GREEN}.{Fore.RESET}")
+    else:
+        output.seek(0)
+    if upload:
+        asyncio.run(upload_plugin(output, server=server))

maubot-src/maubot/cli/commands/init.py

@@ -0,0 +1,99 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import os
+
+from jinja2 import Template
+from packaging.version import Version
+from pkg_resources import resource_string
+
+from .. import cliq
+from ..cliq import SPDXValidator, VersionValidator
+from ..util import spdx
+
+loaded: bool = False
+meta_template: Template
+mod_template: Template
+base_config: str
+
+
+def load_templates():
+    global mod_template, meta_template, base_config, loaded
+    if loaded:
+        return
+    meta_template = Template(resource_string("maubot.cli", "res/maubot.yaml.j2").decode("utf-8"))
+    mod_template = Template(resource_string("maubot.cli", "res/plugin.py.j2").decode("utf-8"))
+    base_config = resource_string("maubot.cli", "res/config.yaml").decode("utf-8")
+    loaded = True
+
+
+@cliq.command(help="Initialize a new maubot plugin")
+@cliq.option(
+    "-n",
+    "--name",
+    help="The name of the project",
+    required=True,
+    default=os.path.basename(os.getcwd()),
+)
+@cliq.option(
+    "-i",
+    "--id",
+    message="ID",
+    required=True,
+    help="The maubot plugin ID (Java package name format)",
+)
+@cliq.option(
+    "-v",
+    "--version",
+    help="Initial version for project (PEP-440 format)",
+    default="0.1.0",
+    validator=VersionValidator,
+    required=True,
+)
+@cliq.option(
+    "-l",
+    "--license",
+    validator=SPDXValidator,
+    default="AGPL-3.0-or-later",
+    help="The license for the project (SPDX identifier)",
+    required=False,
+)
+@cliq.option(
+    "-c",
+    "--config",
+    message="Should the plugin include a config?",
+    help="Include a config in the plugin stub",
+    default=False,
+    is_flag=True,
+)
+def init(name: str, id: str, version: Version, license: str, config: bool) -> None:
+    load_templates()
+    main_class = name[0].upper() + name[1:]
+    meta = meta_template.render(
+        id=id, version=str(version), license=license, config=config, main_class=main_class
+    )
+    with open("maubot.yaml", "w") as file:
+        file.write(meta)
+    if license:
+        with open("LICENSE", "w") as file:
+            file.write(spdx.get(license)["licenseText"])
+    if not os.path.isdir(name):
+        os.mkdir(name)
+    mod = mod_template.render(config=config, name=main_class)
+    with open(f"{name}/__init__.py", "w") as file:
+        file.write(mod)
+    if config:
+        with open("base-config.yaml", "w") as file:
+            file.write(base_config)

maubot-src/maubot/cli/commands/login.py

@@ -0,0 +1,77 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import json
+import os
+
+from colorama import Fore
+from yarl import URL
+import aiohttp
+
+from ..cliq import cliq
+from ..config import config, save_config
+
+
+@cliq.command(help="Log in to a Maubot instance")
+@cliq.option(
+    "-u",
+    "--username",
+    help="The username of your account",
+    default=os.environ.get("USER", None),
+    required=True,
+)
+@cliq.option(
+    "-p", "--password", help="The password to your account", inq_type="password", required=True
+)
+@cliq.option(
+    "-s",
+    "--server",
+    help="The server to log in to",
+    default="http://localhost:29316",
+    required=True,
+)
+@cliq.option(
+    "-a",
+    "--alias",
+    help="Alias to reference the server without typing the full URL",
+    default="",
+    required=False,
+)
+@cliq.with_http
+async def login(
+    server: str, username: str, password: str, alias: str, sess: aiohttp.ClientSession
+) -> None:
+    data = {
+        "username": username,
+        "password": password,
+    }
+    url = URL(server) / "_matrix/maubot/v1/auth/login"
+    async with sess.post(url, json=data) as resp:
+        if resp.status == 200:
+            data = await resp.json()
+            config["servers"][server] = data["token"]
+            if not config["default_server"]:
+                print(Fore.CYAN, "Setting", server, "as the default server")
+                config["default_server"] = server
+            if alias:
+                config["aliases"][alias] = server
+            save_config()
+            print(Fore.GREEN + "Logged in successfully")
+        else:
+            try:
+                err = (await resp.json())["error"]
+            except (json.JSONDecodeError, KeyError):
+                err = await resp.text()
+            print(Fore.RED + err + Fore.RESET)

maubot-src/maubot/cli/commands/logs.py

@@ -0,0 +1,107 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from datetime import datetime
+import asyncio
+
+from aiohttp import ClientSession, WSMessage, WSMsgType
+from colorama import Fore
+import click
+
+from mautrix.types import Obj
+
+from ..base import app
+from ..config import get_token
+
+history_count: int = 10
+
+
+@app.command(help="View the logs of a server")
+@click.argument("server", required=False)
+@click.option("-t", "--tail", default=10, help="Maximum number of old log lines to display")
+def logs(server: str, tail: int) -> None:
+    server, token = get_token(server)
+    if not token:
+        return
+    global history_count
+    history_count = tail
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(view_logs(server, token))
+
+
+def parsedate(entry: Obj) -> None:
+    i = entry.time.index("+")
+    i = entry.time.index(":", i)
+    entry.time = entry.time[:i] + entry.time[i + 1 :]
+    entry.time = datetime.strptime(entry.time, "%Y-%m-%dT%H:%M:%S.%f%z")
+
+
+levelcolors = {
+    "DEBUG": "",
+    "INFO": Fore.CYAN,
+    "WARNING": Fore.YELLOW,
+    "ERROR": Fore.RED,
+    "FATAL": Fore.MAGENTA,
+}
+
+
+def print_entry(entry: dict) -> None:
+    entry = Obj(**entry)
+    parsedate(entry)
+    print(
+        "{levelcolor}[{date}] [{level}@{logger}] {message}{resetcolor}".format(
+            date=entry.time.strftime("%Y-%m-%d %H:%M:%S"),
+            level=entry.levelname,
+            levelcolor=levelcolors.get(entry.levelname, ""),
+            resetcolor=Fore.RESET,
+            logger=entry.name,
+            message=entry.msg,
+        )
+    )
+    if entry.exc_info:
+        print(entry.exc_info)
+
+
+def handle_msg(data: dict) -> bool:
+    if "auth_success" in data:
+        if data["auth_success"]:
+            print(Fore.GREEN + "Connected to log websocket" + Fore.RESET)
+        else:
+            print(Fore.RED + "Failed to authenticate to log websocket" + Fore.RESET)
+            return False
+    elif "history" in data:
+        for entry in data["history"][-history_count:]:
+            print_entry(entry)
+    else:
+        print_entry(data)
+    return True
+
+
+async def view_logs(server: str, token: str) -> None:
+    async with ClientSession() as session:
+        async with session.ws_connect(f"{server}/_matrix/maubot/v1/logs") as ws:
+            await ws.send_str(token)
+            try:
+                msg: WSMessage
+                async for msg in ws:
+                    if msg.type == WSMsgType.TEXT:
+                        if not handle_msg(msg.json()):
+                            break
+                    elif msg.type == WSMsgType.ERROR:
+                        print(Fore.YELLOW + "Connection error: " + msg.data + Fore.RESET)
+                    elif msg.type == WSMsgType.CLOSE:
+                        print(Fore.YELLOW + "Server closed connection" + Fore.RESET)
+            except asyncio.CancelledError:
+                pass

maubot-src/maubot/cli/commands/upload.py

@@ -0,0 +1,58 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import IO
+import json
+
+from colorama import Fore
+from yarl import URL
+import aiohttp
+import click
+
+from ..cliq import cliq
+
+
+class UploadError(Exception):
+    pass
+
+
+@cliq.command(help="Upload a maubot plugin")
+@click.argument("path")
+@click.option("-s", "--server", help="The maubot instance to upload the plugin to")
+@cliq.with_authenticated_http
+async def upload(path: str, server: str, sess: aiohttp.ClientSession) -> None:
+    with open(path, "rb") as file:
+        await upload_file(sess, file, server)
+
+
+async def upload_file(sess: aiohttp.ClientSession, file: IO, server: str) -> None:
+    url = (URL(server) / "_matrix/maubot/v1/plugins/upload").with_query({"allow_override": "true"})
+    headers = {"Content-Type": "application/zip"}
+    async with sess.post(url, data=file, headers=headers) as resp:
+        if resp.status in (200, 201):
+            data = await resp.json()
+            print(
+                f"{Fore.GREEN}Plugin {Fore.CYAN}{data['id']} v{data['version']}{Fore.GREEN} "
+                f"uploaded to {Fore.CYAN}{server}{Fore.GREEN} successfully.{Fore.RESET}"
+            )
+        else:
+            try:
+                err = await resp.json()
+                if "stacktrace" in err:
+                    print(err["stacktrace"])
+                err = err["error"]
+            except (aiohttp.ContentTypeError, json.JSONDecodeError, KeyError):
+                err = await resp.text()
+            print(f"{Fore.RED}Failed to upload plugin: {err}{Fore.RESET}")

maubot-src/maubot/cli/config.py

@@ -0,0 +1,80 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any
+import json
+import os
+
+from colorama import Fore
+
+config: dict[str, Any] = {
+    "servers": {},
+    "aliases": {},
+    "default_server": None,
+}
+configdir = os.environ.get("XDG_CONFIG_HOME", os.path.join(os.environ.get("HOME"), ".config"))
+
+
+def get_default_server() -> tuple[str | None, str | None]:
+    try:
+        server: str < None = config["default_server"]
+    except KeyError:
+        server = None
+    if server is None:
+        print(f"{Fore.RED}Default server not configured.{Fore.RESET}")
+        print(f"Perhaps you forgot to {Fore.CYAN}mbc login{Fore.RESET}?")
+        return None, None
+    return server, _get_token(server)
+
+
+def get_token(server: str) -> tuple[str | None, str | None]:
+    if not server:
+        return get_default_server()
+    if server in config["aliases"]:
+        server = config["aliases"][server]
+    return server, _get_token(server)
+
+
+def _resolve_alias(alias: str) -> str | None:
+    try:
+        return config["aliases"][alias]
+    except KeyError:
+        return None
+
+
+def _get_token(server: str) -> str | None:
+    try:
+        return config["servers"][server]
+    except KeyError:
+        print(f"{Fore.RED}No access token saved for {server}.{Fore.RESET}")
+        return None
+
+
+def save_config() -> None:
+    with open(f"{configdir}/maubot-cli.json", "w") as file:
+        json.dump(config, file)
+
+
+def load_config() -> None:
+    try:
+        with open(f"{configdir}/maubot-cli.json") as file:
+            loaded = json.load(file)
+            config["servers"] = loaded.get("servers", {})
+            config["aliases"] = loaded.get("aliases", {})
+            config["default_server"] = loaded.get("default_server", None)
+    except FileNotFoundError:
+        pass

maubot-src/maubot/cli/res/config.yaml

@@ -0,0 +1,6 @@
+example_1: Example value 1
+example_2:
+    list:
+    - foo
+    - bar
+    value: asd

maubot-src/maubot/cli/res/maubot.yaml.j2

@@ -0,0 +1,42 @@
+# The unique ID for the plugin. Java package naming style. (i.e. use your own domain, not xyz.maubot)
+id: {{ id }}
+
+# A PEP 440 compliant version string.
+version: {{ version }}
+
+# The SPDX license identifier for the plugin. https://spdx.org/licenses/
+# Optional, assumes all rights reserved if omitted.
+{% if license %}
+license: {{ license }}
+{% else %}
+#license: null
+{% endif %}
+
+# The list of modules to load from the plugin archive.
+# Modules can be directories with an __init__.py file or simply python files.
+# Submodules that are imported by modules listed here don't need to be listed separately.
+# However, top-level modules must always be listed even if they're imported by other modules.
+modules:
+- {{ name }}
+
+# The main class of the plugin. Format: module/Class
+# If `module` is omitted, will default to last module specified in the module list.
+# Even if `module` is not omitted here, it must be included in the modules list.
+# The main class must extend maubot.Plugin
+main_class: {{ main_class }}
+
+#  Extra files that the upcoming build tool should include in the mbp file.
+{% if config %}
+extra_files:
+- base-config.yaml
+{% else %}
+#extra_files:
+#- base-config.yaml
+{% endif %}
+
+# List of dependencies
+#dependencies:
+#- foo
+
+#soft_dependencies:
+#- bar>=0.1

maubot-src/maubot/cli/res/plugin.py.j2

@@ -0,0 +1,27 @@
+from typing import Type
+{% if config %}
+from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
+{% endif %}
+from maubot import Plugin
+{% if config %}
+
+class Config(BaseProxyConfig):
+    def do_update(self, helper: ConfigUpdateHelper) -> None:
+        helper.copy("example_1")
+        helper.copy("example_2.list")
+        helper.copy("example_2.value")
+{% endif %}
+
+class {{ name }}(Plugin):
+    async def start(self) -> None:{% if config %}
+        self.config.load_and_update()
+        self.log.debug("Loaded %s from config example 2", self.config["example_2.value"]){% else %}
+        pass{% endif %}
+
+    async def stop(self) -> None:
+        pass
+{% if config %}
+    @classmethod
+    def get_config_class(cls) -> Type[BaseProxyConfig]:
+        return Config
+{% endif %}

maubot-src/maubot/cli/util/spdx.py

@@ -0,0 +1,45 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import json
+import zipfile
+
+import pkg_resources
+
+spdx_list: dict[str, dict[str, str]] | None = None
+
+
+def load() -> None:
+    global spdx_list
+    if spdx_list is not None:
+        return
+    with pkg_resources.resource_stream("maubot.cli", "res/spdx.json.zip") as disk_file:
+        with zipfile.ZipFile(disk_file) as zip_file:
+            with zip_file.open("spdx.json") as file:
+                spdx_list = json.load(file)
+
+
+def get(id: str) -> dict[str, str]:
+    if not spdx_list:
+        load()
+    return spdx_list[id]
+
+
+def valid(id: str) -> bool:
+    if not spdx_list:
+        load()
+    return id in spdx_list

maubot-src/maubot/client.py

@@ -0,0 +1,581 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, cast
+from collections import defaultdict
+import asyncio
+import logging
+
+from aiohttp import ClientSession
+
+from mautrix.api import HTTPAPI
+from mautrix.client import InternalEventType
+from mautrix.errors import MatrixInvalidToken
+from mautrix.types import (
+    ContentURI,
+    DeviceID,
+    EventFilter,
+    EventType,
+    Filter,
+    FilterID,
+    Membership,
+    PresenceState,
+    RoomEventFilter,
+    RoomFilter,
+    StateEvent,
+    StateFilter,
+    StrippedStateEvent,
+    SyncToken,
+    TrustState,
+    UserID,
+)
+from mautrix.util import background_task
+from mautrix.util.async_getter_lock import async_getter_lock
+from mautrix.util.logging import TraceLogger
+
+from .db import Client as DBClient
+from .matrix import MaubotMatrixClient
+
+try:
+    from mautrix.crypto import OlmMachine, PgCryptoStore
+
+    crypto_import_error = None
+except ImportError as e:
+    OlmMachine = PgCryptoStore = None
+    crypto_import_error = e
+
+if TYPE_CHECKING:
+    from .__main__ import Maubot
+    from .instance import PluginInstance
+
+
+class Client(DBClient):
+    maubot: "Maubot" = None
+    cache: dict[UserID, Client] = {}
+    _async_get_locks: dict[Any, asyncio.Lock] = defaultdict(lambda: asyncio.Lock())
+    log: TraceLogger = logging.getLogger("maubot.client")
+
+    http_client: ClientSession = None
+
+    references: set[PluginInstance]
+    client: MaubotMatrixClient
+    crypto: OlmMachine | None
+    crypto_store: PgCryptoStore | None
+    started: bool
+    sync_ok: bool
+
+    remote_displayname: str | None
+    remote_avatar_url: ContentURI | None
+    trust_state: TrustState | None
+
+    def __init__(
+        self,
+        id: UserID,
+        homeserver: str,
+        access_token: str,
+        device_id: DeviceID,
+        enabled: bool = False,
+        next_batch: SyncToken = "",
+        filter_id: FilterID = "",
+        sync: bool = True,
+        autojoin: bool = True,
+        online: bool = True,
+        displayname: str = "disable",
+        avatar_url: str = "disable",
+    ) -> None:
+        super().__init__(
+            id=id,
+            homeserver=homeserver,
+            access_token=access_token,
+            device_id=device_id,
+            enabled=bool(enabled),
+            next_batch=next_batch,
+            filter_id=filter_id,
+            sync=bool(sync),
+            autojoin=bool(autojoin),
+            online=bool(online),
+            displayname=displayname,
+            avatar_url=avatar_url,
+        )
+        self._postinited = False
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    @classmethod
+    def init_cls(cls, maubot: "Maubot") -> None:
+        cls.maubot = maubot
+
+    def _make_client(
+        self, homeserver: str | None = None, token: str | None = None, device_id: str | None = None
+    ) -> MaubotMatrixClient:
+        return MaubotMatrixClient(
+            mxid=self.id,
+            base_url=homeserver or self.homeserver,
+            token=token or self.access_token,
+            client_session=self.http_client,
+            log=self.log,
+            crypto_log=self.log.getChild("crypto"),
+            loop=self.maubot.loop,
+            device_id=device_id or self.device_id,
+            sync_store=self,
+            state_store=self.maubot.state_store,
+        )
+
+    def postinit(self) -> None:
+        if self._postinited:
+            raise RuntimeError("postinit() called twice")
+        self._postinited = True
+        self.cache[self.id] = self
+        self.log = self.log.getChild(self.id)
+        self.http_client = ClientSession(
+            loop=self.maubot.loop,
+            headers={
+                "User-Agent": HTTPAPI.default_ua,
+            },
+        )
+        self.references = set()
+        self.started = False
+        self.sync_ok = True
+        self.trust_state = None
+        self.remote_displayname = None
+        self.remote_avatar_url = None
+        self.client = self._make_client()
+        if self.enable_crypto:
+            self._prepare_crypto()
+        else:
+            self.crypto_store = None
+            self.crypto = None
+        self.client.ignore_initial_sync = True
+        self.client.ignore_first_sync = True
+        self.client.presence = PresenceState.ONLINE if self.online else PresenceState.OFFLINE
+        if self.autojoin:
+            self.client.add_event_handler(EventType.ROOM_MEMBER, self._handle_invite)
+        self.client.add_event_handler(EventType.ROOM_TOMBSTONE, self._handle_tombstone)
+        self.client.add_event_handler(InternalEventType.SYNC_ERRORED, self._set_sync_ok(False))
+        self.client.add_event_handler(InternalEventType.SYNC_SUCCESSFUL, self._set_sync_ok(True))
+
+    def _set_sync_ok(self, ok: bool) -> Callable[[dict[str, Any]], Awaitable[None]]:
+        async def handler(data: dict[str, Any]) -> None:
+            self.sync_ok = ok
+
+        return handler
+
+    @property
+    def enable_crypto(self) -> bool:
+        if not self.device_id:
+            return False
+        elif not OlmMachine:
+            global crypto_import_error
+            self.log.warning(
+                "Client has device ID, but encryption dependencies not installed",
+                exc_info=crypto_import_error,
+            )
+            # Clear the stack trace after it's logged once to avoid spamming logs
+            crypto_import_error = None
+            return False
+        elif not self.maubot.crypto_db:
+            self.log.warning("Client has device ID, but crypto database is not prepared")
+            return False
+        return True
+
+    def _prepare_crypto(self) -> None:
+        self.crypto_store = PgCryptoStore(
+            account_id=self.id, pickle_key="mau.crypto", db=self.maubot.crypto_db
+        )
+        self.crypto = OlmMachine(
+            self.client,
+            self.crypto_store,
+            self.maubot.state_store,
+            log=self.client.crypto_log,
+        )
+        self.client.crypto = self.crypto
+
+    def _remove_crypto_event_handlers(self) -> None:
+        if not self.crypto:
+            return
+        handlers = [
+            (InternalEventType.DEVICE_OTK_COUNT, self.crypto.handle_otk_count),
+            (InternalEventType.DEVICE_LISTS, self.crypto.handle_device_lists),
+            (EventType.TO_DEVICE_ENCRYPTED, self.crypto.handle_to_device_event),
+            (EventType.ROOM_KEY_REQUEST, self.crypto.handle_room_key_request),
+            (EventType.ROOM_MEMBER, self.crypto.handle_member_event),
+        ]
+        for event_type, func in handlers:
+            self.client.remove_event_handler(event_type, func)
+
+    async def start(self, try_n: int | None = 0) -> None:
+        try:
+            if try_n > 0:
+                await asyncio.sleep(try_n * 10)
+            await self._start(try_n)
+        except Exception:
+            self.log.exception("Failed to start")
+
+    async def _start_crypto(self) -> None:
+        self.log.debug("Enabling end-to-end encryption support")
+        await self.crypto_store.open()
+        crypto_device_id = await self.crypto_store.get_device_id()
+        if crypto_device_id and crypto_device_id != self.device_id:
+            self.log.warning(
+                "Mismatching device ID in crypto store and main database, resetting encryption"
+            )
+            await self.crypto_store.delete()
+            crypto_device_id = None
+        await self.crypto.load()
+        if not crypto_device_id:
+            await self.crypto_store.put_device_id(self.device_id)
+        if not self.crypto.account.shared:
+            await self.crypto.share_keys()
+        self.trust_state = await self.crypto.resolve_trust(
+            self.crypto.own_identity,
+            allow_fetch=False,
+        )
+
+    async def _start(self, try_n: int | None = 0) -> None:
+        if not self.enabled:
+            self.log.debug("Not starting disabled client")
+            return
+        elif self.started:
+            self.log.warning("Ignoring start() call to started client")
+            return
+        try:
+            await self.client.versions()
+            whoami = await self.client.whoami()
+        except MatrixInvalidToken as e:
+            self.log.error(f"Invalid token: {e}. Disabling client")
+            self.enabled = False
+            await self.update()
+            return
+        except Exception as e:
+            if try_n >= 8:
+                self.log.exception("Failed to get /account/whoami, disabling client")
+                self.enabled = False
+                await self.update()
+            else:
+                self.log.warning(
+                    f"Failed to get /account/whoami, retrying in {(try_n + 1) * 10}s: {e}"
+                )
+                background_task.create(self.start(try_n + 1))
+            return
+        if whoami.user_id != self.id:
+            self.log.error(f"User ID mismatch: expected {self.id}, but got {whoami.user_id}")
+            self.enabled = False
+            await self.update()
+            return
+        elif whoami.device_id and self.device_id and whoami.device_id != self.device_id:
+            self.log.error(
+                f"Device ID mismatch: expected {self.device_id}, but got {whoami.device_id}"
+            )
+            self.enabled = False
+            await self.update()
+            return
+        if not self.filter_id:
+            self.filter_id = await self.client.create_filter(
+                Filter(
+                    room=RoomFilter(
+                        timeline=RoomEventFilter(
+                            limit=50,
+                            lazy_load_members=True,
+                        ),
+                        state=StateFilter(
+                            lazy_load_members=True,
+                        ),
+                    ),
+                    presence=EventFilter(
+                        not_types=[EventType.PRESENCE],
+                    ),
+                )
+            )
+            await self.update()
+        if self.displayname != "disable":
+            await self.client.set_displayname(self.displayname)
+        if self.avatar_url != "disable":
+            await self.client.set_avatar_url(self.avatar_url)
+        if self.crypto:
+            await self._start_crypto()
+        self.start_sync()
+        await self._update_remote_profile()
+        self.started = True
+        self.log.info("Client started, starting plugin instances...")
+        await self.start_plugins()
+
+    async def start_plugins(self) -> None:
+        await asyncio.gather(*[plugin.start() for plugin in self.references])
+
+    async def stop_plugins(self) -> None:
+        await asyncio.gather(*[plugin.stop() for plugin in self.references if plugin.started])
+
+    def start_sync(self) -> None:
+        if self.sync:
+            self.client.start(self.filter_id)
+
+    def stop_sync(self) -> None:
+        self.client.stop()
+
+    async def stop(self) -> None:
+        if self.started:
+            self.started = False
+            await self.stop_plugins()
+            self.stop_sync()
+            if self.crypto:
+                await self.crypto_store.close()
+
+    async def clear_cache(self) -> None:
+        self.stop_sync()
+        self.filter_id = FilterID("")
+        self.next_batch = SyncToken("")
+        await self.update()
+        self.start_sync()
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "homeserver": self.homeserver,
+            "access_token": self.access_token,
+            "device_id": self.device_id,
+            "fingerprint": (
+                self.crypto.account.fingerprint if self.crypto and self.crypto.account else None
+            ),
+            "enabled": self.enabled,
+            "started": self.started,
+            "sync": self.sync,
+            "sync_ok": self.sync_ok,
+            "autojoin": self.autojoin,
+            "online": self.online,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+            "remote_displayname": self.remote_displayname,
+            "remote_avatar_url": self.remote_avatar_url,
+            "instances": [instance.to_dict() for instance in self.references],
+            "trust_state": str(self.trust_state) if self.trust_state else None,
+        }
+
+    async def _handle_tombstone(self, evt: StateEvent) -> None:
+        if evt.state_key != "":
+            return
+        if not evt.content.replacement_room:
+            self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring")
+            return
+        is_joined = await self.client.state_store.is_joined(
+            evt.content.replacement_room,
+            self.client.mxid,
+        )
+        if is_joined:
+            self.log.debug(
+                f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} "
+                f"sent by {evt.sender}: already joined to replacement room"
+            )
+            return
+        self.log.debug(
+            f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} "
+            f"sent by {evt.sender}"
+        )
+        _, server = self.client.parse_user_id(evt.sender)
+        room_id = await self.client.join_room(evt.content.replacement_room, servers=[server])
+        power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS)
+        create_event = await self.client.get_state_event(
+            room_id, EventType.ROOM_CREATE, format="event"
+        )
+        if power_levels.get_user_level(evt.sender, create_event) < power_levels.invite:
+            self.log.warning(
+                f"{evt.room_id} was tombstoned into {room_id} by {evt.sender},"
+                " but the sender doesn't have invite power levels, leaving..."
+            )
+            await self.client.leave_room(
+                room_id,
+                f"Followed tombstone from {evt.room_id} by {evt.sender},"
+                " but sender doesn't have sufficient power level for invites",
+            )
+
+    async def _handle_invite(self, evt: StrippedStateEvent) -> None:
+        if evt.state_key == self.id and evt.content.membership == Membership.INVITE:
+            await self.client.join_room(evt.room_id)
+
+    async def update_started(self, started: bool | None) -> None:
+        if started is None or started == self.started:
+            return
+        if started:
+            await self.start()
+        else:
+            await self.stop()
+
+    async def update_enabled(self, enabled: bool | None, save: bool = True) -> None:
+        if enabled is None or enabled == self.enabled:
+            return
+        self.enabled = enabled
+        if save:
+            await self.update()
+
+    async def update_displayname(self, displayname: str | None, save: bool = True) -> None:
+        if displayname is None or displayname == self.displayname:
+            return
+        self.displayname = displayname
+        if self.displayname != "disable":
+            await self.client.set_displayname(self.displayname)
+        else:
+            await self._update_remote_profile()
+        if save:
+            await self.update()
+
+    async def update_avatar_url(self, avatar_url: ContentURI, save: bool = True) -> None:
+        if avatar_url is None or avatar_url == self.avatar_url:
+            return
+        self.avatar_url = avatar_url
+        if self.avatar_url != "disable":
+            await self.client.set_avatar_url(self.avatar_url)
+        else:
+            await self._update_remote_profile()
+        if save:
+            await self.update()
+
+    async def update_sync(self, sync: bool | None, save: bool = True) -> None:
+        if sync is None or self.sync == sync:
+            return
+        self.sync = sync
+        if self.started:
+            if sync:
+                self.start_sync()
+            else:
+                self.stop_sync()
+        if save:
+            await self.update()
+
+    async def update_autojoin(self, autojoin: bool | None, save: bool = True) -> None:
+        if autojoin is None or autojoin == self.autojoin:
+            return
+        if autojoin:
+            self.client.add_event_handler(EventType.ROOM_MEMBER, self._handle_invite)
+        else:
+            self.client.remove_event_handler(EventType.ROOM_MEMBER, self._handle_invite)
+        self.autojoin = autojoin
+        if save:
+            await self.update()
+
+    async def update_online(self, online: bool | None, save: bool = True) -> None:
+        if online is None or online == self.online:
+            return
+        self.client.presence = PresenceState.ONLINE if online else PresenceState.OFFLINE
+        self.online = online
+        if save:
+            await self.update()
+
+    async def update_access_details(
+        self,
+        access_token: str | None,
+        homeserver: str | None,
+        device_id: str | None = None,
+    ) -> None:
+        if not access_token and not homeserver:
+            return
+        if device_id is None:
+            device_id = self.device_id
+        elif not device_id:
+            device_id = None
+        if (
+            access_token == self.access_token
+            and homeserver == self.homeserver
+            and device_id == self.device_id
+        ):
+            return
+        new_client = self._make_client(homeserver, access_token, device_id)
+        whoami = await new_client.whoami()
+        if whoami.user_id != self.id:
+            raise ValueError(f"MXID mismatch: {whoami.user_id}")
+        elif whoami.device_id and device_id and whoami.device_id != device_id:
+            raise ValueError(f"Device ID mismatch: {whoami.device_id}")
+        new_client.sync_store = self
+        self.stop_sync()
+
+        # TODO this event handler transfer is pretty hacky
+        self._remove_crypto_event_handlers()
+        self.client.crypto = None
+        new_client.event_handlers = self.client.event_handlers
+        new_client.global_event_handlers = self.client.global_event_handlers
+
+        self.client = new_client
+        self.homeserver = homeserver
+        self.access_token = access_token
+        self.device_id = device_id
+        if self.enable_crypto:
+            self._prepare_crypto()
+            await self._start_crypto()
+        else:
+            self.crypto_store = None
+            self.crypto = None
+        self.start_sync()
+
+    async def _update_remote_profile(self) -> None:
+        try:
+            profile = await self.client.get_profile(self.id)
+            self.remote_displayname, self.remote_avatar_url = (
+                profile.displayname,
+                profile.avatar_url,
+            )
+        except Exception:
+            self.log.warning("Failed to update own profile from server", exc_info=True)
+
+    async def delete(self) -> None:
+        try:
+            del self.cache[self.id]
+        except KeyError:
+            pass
+        await super().delete()
+
+    @classmethod
+    @async_getter_lock
+    async def get(
+        cls,
+        user_id: UserID,
+        *,
+        homeserver: str | None = None,
+        access_token: str | None = None,
+        device_id: DeviceID | None = None,
+    ) -> Client | None:
+        try:
+            return cls.cache[user_id]
+        except KeyError:
+            pass
+
+        user = cast(cls, await super().get(user_id))
+        if user is not None:
+            user.postinit()
+            return user
+
+        if homeserver and access_token:
+            user = cls(
+                user_id,
+                homeserver=homeserver,
+                access_token=access_token,
+                device_id=device_id or "",
+            )
+            await user.insert()
+            user.postinit()
+            return user
+
+        return None
+
+    @classmethod
+    async def all(cls) -> AsyncGenerator[Client, None]:
+        users = await super().all()
+        user: cls
+        for user in users:
+            try:
+                yield cls.cache[user.id]
+            except KeyError:
+                user.postinit()
+                yield user

maubot-src/maubot/config.py

@@ -0,0 +1,100 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import random
+import re
+import string
+
+import bcrypt
+
+from mautrix.util.config import BaseFileConfig, ConfigUpdateHelper
+
+bcrypt_regex = re.compile(r"^\$2[ayb]\$.{56}$")
+
+
+class Config(BaseFileConfig):
+    @staticmethod
+    def _new_token() -> str:
+        return "".join(random.choices(string.ascii_lowercase + string.digits, k=64))
+
+    def do_update(self, helper: ConfigUpdateHelper) -> None:
+        base = helper.base
+        copy = helper.copy
+
+        if "database" in self and self["database"].startswith("sqlite:///"):
+            helper.base["database"] = self["database"].replace("sqlite:///", "sqlite:")
+        else:
+            copy("database")
+        copy("database_opts")
+        if isinstance(self["crypto_database"], dict):
+            if self["crypto_database.type"] == "postgres":
+                base["crypto_database"] = self["crypto_database.postgres_uri"]
+        else:
+            copy("crypto_database")
+        copy("plugin_directories.upload")
+        copy("plugin_directories.load")
+        copy("plugin_directories.trash")
+        if "plugin_directories.db" in self:
+            base["plugin_databases.sqlite"] = self["plugin_directories.db"]
+        else:
+            copy("plugin_databases.sqlite")
+        copy("plugin_databases.postgres")
+        copy("plugin_databases.postgres_opts")
+        copy("server.hostname")
+        copy("server.port")
+        copy("server.public_url")
+        copy("server.listen")
+        copy("server.ui_base_path")
+        copy("server.plugin_base_path")
+        copy("server.override_resource_path")
+        shared_secret = self["server.unshared_secret"]
+        if shared_secret is None or shared_secret == "generate":
+            base["server.unshared_secret"] = self._new_token()
+        else:
+            base["server.unshared_secret"] = shared_secret
+        if "registration_secrets" in self:
+            base["homeservers"] = self["registration_secrets"]
+        else:
+            copy("homeservers")
+        copy("admins")
+        for username, password in base["admins"].items():
+            if password and not bcrypt_regex.match(password):
+                if password == "password":
+                    password = self._new_token()
+                base["admins"][username] = bcrypt.hashpw(
+                    password.encode("utf-8"), bcrypt.gensalt()
+                ).decode("utf-8")
+        copy("api_features.login")
+        copy("api_features.plugin")
+        copy("api_features.plugin_upload")
+        copy("api_features.instance")
+        copy("api_features.instance_database")
+        copy("api_features.client")
+        copy("api_features.client_proxy")
+        copy("api_features.client_auth")
+        copy("api_features.dev_open")
+        copy("api_features.log")
+        copy("logging")
+
+    def is_admin(self, user: str) -> bool:
+        return user == "root" or user in self["admins"]
+
+    def check_password(self, user: str, passwd: str) -> bool:
+        if user == "root":
+            return False
+        passwd_hash = self["admins"].get(user, None)
+        if not passwd_hash:
+            return False
+        return bcrypt.checkpw(passwd.encode("utf-8"), passwd_hash.encode("utf-8"))

maubot-src/maubot/db/__init__.py

@@ -0,0 +1,13 @@
+from mautrix.util.async_db import Database
+
+from .client import Client
+from .instance import DatabaseEngine, Instance
+from .upgrade import upgrade_table
+
+
+def init(db: Database) -> None:
+    for table in (Client, Instance):
+        table.db = db
+
+
+__all__ = ["upgrade_table", "init", "Client", "Instance", "DatabaseEngine"]

maubot-src/maubot/db/client.py

@@ -0,0 +1,114 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.client import SyncStore
+from mautrix.types import ContentURI, DeviceID, FilterID, SyncToken, UserID
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Client(SyncStore):
+    db: ClassVar[Database] = fake_db
+
+    id: UserID
+    homeserver: str
+    access_token: str
+    device_id: DeviceID
+    enabled: bool
+
+    next_batch: SyncToken
+    filter_id: FilterID
+
+    sync: bool
+    autojoin: bool
+    online: bool
+
+    displayname: str
+    avatar_url: ContentURI
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Client | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    _columns = (
+        "id, homeserver, access_token, device_id, enabled, next_batch, filter_id, "
+        "sync, autojoin, online, displayname, avatar_url"
+    )
+
+    @property
+    def _values(self):
+        return (
+            self.id,
+            self.homeserver,
+            self.access_token,
+            self.device_id,
+            self.enabled,
+            self.next_batch,
+            self.filter_id,
+            self.sync,
+            self.autojoin,
+            self.online,
+            self.displayname,
+            self.avatar_url,
+        )
+
+    @classmethod
+    async def all(cls) -> list[Client]:
+        rows = await cls.db.fetch(f"SELECT {cls._columns} FROM client")
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get(cls, id: str) -> Client | None:
+        q = f"SELECT {cls._columns} FROM client WHERE id=$1"
+        return cls._from_row(await cls.db.fetchrow(q, id))
+
+    async def insert(self) -> None:
+        q = """
+        INSERT INTO client (
+            id, homeserver, access_token, device_id, enabled, next_batch, filter_id,
+            sync, autojoin, online, displayname, avatar_url
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+        """
+        await self.db.execute(q, *self._values)
+
+    async def put_next_batch(self, next_batch: SyncToken) -> None:
+        await self.db.execute("UPDATE client SET next_batch=$1 WHERE id=$2", next_batch, self.id)
+        self.next_batch = next_batch
+
+    async def get_next_batch(self) -> SyncToken:
+        return self.next_batch
+
+    async def update(self) -> None:
+        q = """
+        UPDATE client SET homeserver=$2, access_token=$3, device_id=$4, enabled=$5,
+                          next_batch=$6, filter_id=$7, sync=$8, autojoin=$9, online=$10,
+                          displayname=$11, avatar_url=$12
+        WHERE id=$1
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        await self.db.execute("DELETE FROM client WHERE id=$1", self.id)

maubot-src/maubot/db/instance.py

@@ -0,0 +1,101 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+from enum import Enum
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import UserID
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class DatabaseEngine(Enum):
+    SQLITE = "sqlite"
+    POSTGRES = "postgres"
+
+
+@dataclass
+class Instance:
+    db: ClassVar[Database] = fake_db
+
+    id: str
+    type: str
+    enabled: bool
+    primary_user: UserID
+    config_str: str
+    database_engine: DatabaseEngine | None
+
+    @property
+    def database_engine_str(self) -> str | None:
+        return self.database_engine.value if self.database_engine else None
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Instance | None:
+        if row is None:
+            return None
+        data = {**row}
+        db_engine = data.pop("database_engine", None)
+        return cls(**data, database_engine=DatabaseEngine(db_engine) if db_engine else None)
+
+    _columns = "id, type, enabled, primary_user, config, database_engine"
+
+    @classmethod
+    async def all(cls) -> list[Instance]:
+        q = f"SELECT {cls._columns} FROM instance"
+        rows = await cls.db.fetch(q)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get(cls, id: str) -> Instance | None:
+        q = f"SELECT {cls._columns} FROM instance WHERE id=$1"
+        return cls._from_row(await cls.db.fetchrow(q, id))
+
+    async def update_id(self, new_id: str) -> None:
+        await self.db.execute("UPDATE instance SET id=$1 WHERE id=$2", new_id, self.id)
+        self.id = new_id
+
+    @property
+    def _values(self):
+        return (
+            self.id,
+            self.type,
+            self.enabled,
+            self.primary_user,
+            self.config_str,
+            self.database_engine_str,
+        )
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO instance (id, type, enabled, primary_user, config, database_engine) "
+            "VALUES ($1, $2, $3, $4, $5, $6)"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def update(self) -> None:
+        q = """
+        UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5, database_engine=$6
+        WHERE id=$1
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        await self.db.execute("DELETE FROM instance WHERE id=$1", self.id)

maubot-src/maubot/db/upgrade/__init__.py

@@ -0,0 +1,5 @@
+from mautrix.util.async_db import UpgradeTable
+
+upgrade_table = UpgradeTable()
+
+from . import v01_initial_revision, v02_instance_database_engine

maubot-src/maubot/db/upgrade/v01_initial_revision.py

@@ -0,0 +1,136 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+legacy_version_query = "SELECT version_num FROM alembic_version"
+last_legacy_version = "90aa88820eab"
+
+
+@upgrade_table.register(description="Initial asyncpg revision")
+async def upgrade_v1(conn: Connection, scheme: Scheme) -> None:
+    if await conn.table_exists("alembic_version"):
+        await migrate_legacy_to_v1(conn, scheme)
+    else:
+        return await create_v1_tables(conn)
+
+
+async def create_v1_tables(conn: Connection) -> None:
+    await conn.execute(
+        """CREATE TABLE client (
+            id           TEXT    PRIMARY KEY,
+            homeserver   TEXT    NOT NULL,
+            access_token TEXT    NOT NULL,
+            device_id    TEXT    NOT NULL,
+            enabled      BOOLEAN NOT NULL,
+
+            next_batch TEXT NOT NULL,
+            filter_id  TEXT NOT NULL,
+
+            sync     BOOLEAN NOT NULL,
+            autojoin BOOLEAN NOT NULL,
+            online   BOOLEAN NOT NULL,
+
+            displayname TEXT NOT NULL,
+            avatar_url  TEXT NOT NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE instance (
+            id           TEXT    PRIMARY KEY,
+            type         TEXT    NOT NULL,
+            enabled      BOOLEAN NOT NULL,
+            primary_user TEXT    NOT NULL,
+            config       TEXT    NOT NULL,
+            FOREIGN KEY (primary_user) REFERENCES client(id) ON DELETE RESTRICT ON UPDATE CASCADE
+        )"""
+    )
+
+
+async def migrate_legacy_to_v1(conn: Connection, scheme: Scheme) -> None:
+    legacy_version = await conn.fetchval(legacy_version_query)
+    if legacy_version != last_legacy_version:
+        raise RuntimeError(
+            "Legacy database is not on last version. "
+            "Please upgrade the old database with alembic or drop it completely first."
+        )
+    await conn.execute("ALTER TABLE plugin RENAME TO instance")
+    await update_state_store(conn, scheme)
+    if scheme != Scheme.SQLITE:
+        await varchar_to_text(conn)
+    await conn.execute("DROP TABLE alembic_version")
+
+
+async def update_state_store(conn: Connection, scheme: Scheme) -> None:
+    # The Matrix state store already has more or less the correct schema, so set the version
+    await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
+    await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
+    if scheme != Scheme.SQLITE:
+        # Remove old uppercase membership type and recreate it as lowercase
+        await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN membership TYPE TEXT")
+        await conn.execute("DROP TYPE IF EXISTS membership")
+        await conn.execute(
+            "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"
+        )
+        await conn.execute(
+            "ALTER TABLE mx_user_profile ALTER COLUMN membership TYPE membership "
+            "USING LOWER(membership)::membership"
+        )
+    else:
+        # Recreate table to remove CHECK constraint and lowercase everything
+        await conn.execute(
+            """CREATE TABLE new_mx_user_profile (
+                room_id     TEXT,
+                user_id     TEXT,
+                membership  TEXT NOT NULL
+                            CHECK (membership IN ('join', 'leave', 'invite', 'ban', 'knock')),
+                displayname TEXT,
+                avatar_url  TEXT,
+                PRIMARY KEY (room_id, user_id)
+            )"""
+        )
+        await conn.execute(
+            """
+            INSERT INTO new_mx_user_profile (room_id, user_id, membership, displayname, avatar_url)
+            SELECT room_id, user_id, LOWER(membership), displayname, avatar_url
+            FROM mx_user_profile
+            """
+        )
+        await conn.execute("DROP TABLE mx_user_profile")
+        await conn.execute("ALTER TABLE new_mx_user_profile RENAME TO mx_user_profile")
+
+
+async def varchar_to_text(conn: Connection) -> None:
+    columns_to_adjust = {
+        "client": (
+            "id",
+            "homeserver",
+            "device_id",
+            "next_batch",
+            "filter_id",
+            "displayname",
+            "avatar_url",
+        ),
+        "instance": ("id", "type", "primary_user"),
+        "mx_room_state": ("room_id",),
+        "mx_user_profile": ("room_id", "user_id", "displayname", "avatar_url"),
+    }
+    for table, columns in columns_to_adjust.items():
+        for column in columns:
+            await conn.execute(f'ALTER TABLE "{table}" ALTER COLUMN {column} TYPE TEXT')

maubot-src/maubot/db/upgrade/v02_instance_database_engine.py

@@ -0,0 +1,25 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store instance database engine")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE instance ADD COLUMN database_engine TEXT")

maubot-src/maubot/example-config.yaml

@@ -0,0 +1,132 @@
+# The full URI to the database. SQLite and Postgres are fully supported.
+# Format examples:
+#   SQLite:   sqlite:filename.db
+#   Postgres: postgresql://username:password@hostname/dbname
+database: sqlite:maubot.db
+
+# Separate database URL for the crypto database. "default" means use the same database as above.
+crypto_database: default
+
+# Additional arguments for asyncpg.create_pool() or sqlite3.connect()
+# https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool
+# https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
+# For sqlite, min_size is used as the connection thread pool size and max_size is ignored.
+database_opts:
+    min_size: 1
+    max_size: 10
+
+# Configuration for storing plugin .mbp files
+plugin_directories:
+    # The directory where uploaded new plugins should be stored.
+    upload: ./plugins
+    # The directories from which plugins should be loaded.
+    # Duplicate plugin IDs will be moved to the trash.
+    load:
+    - ./plugins
+    # The directory where old plugin versions and conflicting plugins should be moved.
+    # Set to "delete" to delete files immediately.
+    trash: ./trash
+
+# Configuration for storing plugin databases
+plugin_databases:
+    # The directory where SQLite plugin databases should be stored.
+    sqlite: ./plugins
+    # The connection URL for plugin databases. If null, all plugins will get SQLite databases.
+    # If set, plugins using the new asyncpg interface will get a Postgres connection instead.
+    # Plugins using the legacy SQLAlchemy interface will always get a SQLite connection.
+    #
+    # To use the same connection pool as the default database, set to "default"
+    # (the default database above must be postgres to do this).
+    #
+    # When enabled, maubot will create separate Postgres schemas in the database for each plugin.
+    # To view schemas in psql, use `\dn`. To view enter and interact with a specific schema,
+    # use `SET search_path = name` (where `name` is the name found with `\dn`) and then use normal
+    # SQL queries/psql commands.
+    postgres: null
+    # Maximum number of connections per plugin instance.
+    postgres_max_conns_per_plugin: 3
+    # Overrides for the default database_opts when using a non-"default" postgres connection string.
+    postgres_opts: {}
+
+server:
+    # The IP and port to listen to.
+    hostname: 0.0.0.0
+    port: 29316
+    # Public base URL where the server is visible.
+    public_url: https://example.com
+    # The base path for the UI. Note that this does not change the API path.
+    # Add a path prefix to public_url if you want everything on a subpath.
+    ui_base_path: /_matrix/maubot
+    # The base path for plugin endpoints. The instance ID will be appended directly.
+    plugin_base_path: /_matrix/maubot/plugin/
+    # Override path from where to load UI resources.
+    # Set to false to using pkg_resources to find the path.
+    override_resource_path: false
+    # The shared secret to sign API access tokens.
+    # Set to "generate" to generate and save a new token at startup.
+    unshared_secret: generate
+
+# Known homeservers. This is required for the `mbc auth` command and also allows
+# more convenient access from the management UI. This is not required to create
+# clients in the management UI, since you can also just type the homeserver URL
+# into the box there.
+homeservers:
+    matrix.org:
+        # Client-server API URL
+        url: https://matrix-client.matrix.org
+        # registration_shared_secret from synapse config
+        # You can leave this empty if you don't have access to the homeserver.
+        # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will.
+        secret: null
+
+# List of administrator users. Each key is a username and the value is the password.
+# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login.
+# Root is a special user that can't have a password and will always exist.
+admins:
+    root: ""
+
+# API feature switches.
+api_features:
+    login: true
+    plugin: true
+    plugin_upload: true
+    instance: true
+    instance_database: true
+    client: true
+    client_proxy: true
+    client_auth: true
+    dev_open: true
+    log: true
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        colored:
+            (): maubot.lib.color_log.ColorFormatter
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+        normal:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: normal
+            filename: ./maubot.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: colored
+    loggers:
+        maubot:
+            level: DEBUG
+        mau:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

maubot-src/maubot/handlers/__init__.py

@@ -0,0 +1 @@
+from . import command, event, web

maubot-src/maubot/handlers/command.py

@@ -0,0 +1,496 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    NewType,
+    Optional,
+    Pattern,
+    Sequence,
+    Set,
+    Tuple,
+    Union,
+)
+from abc import ABC, abstractmethod
+import asyncio
+import functools
+import inspect
+import re
+
+from mautrix.types import EventType, MessageType
+
+from ..matrix import MaubotMessageEvent
+from . import event
+
+PrefixType = Optional[Union[str, Callable[[], str], Callable[[Any], str]]]
+AliasesType = Union[
+    List[str], Tuple[str, ...], Set[str], Callable[[str], bool], Callable[[Any, str], bool]
+]
+CommandHandlerFunc = NewType(
+    "CommandHandlerFunc", Callable[[MaubotMessageEvent, Any], Awaitable[Any]]
+)
+CommandHandlerDecorator = NewType(
+    "CommandHandlerDecorator",
+    Callable[[Union["CommandHandler", CommandHandlerFunc]], "CommandHandler"],
+)
+PassiveCommandHandlerDecorator = NewType(
+    "PassiveCommandHandlerDecorator", Callable[[CommandHandlerFunc], CommandHandlerFunc]
+)
+
+
+def _split_in_two(val: str, split_by: str) -> List[str]:
+    return val.split(split_by, 1) if split_by in val else [val, ""]
+
+
+class CommandHandler:
+    def __init__(self, func: CommandHandlerFunc) -> None:
+        self.__mb_func__: CommandHandlerFunc = func
+        self.__mb_parent__: Optional[CommandHandler] = None
+        self.__mb_subcommands__: List[CommandHandler] = []
+        self.__mb_arguments__: List[Argument] = []
+        self.__mb_help__: Optional[str] = None
+        self.__mb_get_name__: Callable[[Any], str] = lambda s: "noname"
+        self.__mb_is_command_match__: Callable[[Any, str], bool] = self.__command_match_unset
+        self.__mb_require_subcommand__: bool = True
+        self.__mb_must_consume_args__: bool = True
+        self.__mb_arg_fallthrough__: bool = True
+        self.__mb_event_handler__: bool = True
+        self.__mb_event_types__: set[EventType] = {EventType.ROOM_MESSAGE}
+        self.__mb_msgtypes__: Iterable[MessageType] = (MessageType.TEXT,)
+        self.__bound_copies__: Dict[Any, CommandHandler] = {}
+        self.__bound_instance__: Any = None
+
+    def __get__(self, instance, instancetype):
+        if not instance or self.__bound_instance__:
+            return self
+        try:
+            return self.__bound_copies__[instance]
+        except KeyError:
+            new_ch = type(self)(self.__mb_func__)
+            keys = [
+                "parent",
+                "subcommands",
+                "arguments",
+                "help",
+                "get_name",
+                "is_command_match",
+                "require_subcommand",
+                "must_consume_args",
+                "arg_fallthrough",
+                "event_handler",
+                "event_types",
+                "msgtypes",
+            ]
+            for key in keys:
+                key = f"__mb_{key}__"
+                setattr(new_ch, key, getattr(self, key))
+            new_ch.__bound_instance__ = instance
+            new_ch.__mb_subcommands__ = [
+                subcmd.__get__(instance, instancetype) for subcmd in self.__mb_subcommands__
+            ]
+            self.__bound_copies__[instance] = new_ch
+            return new_ch
+
+    @staticmethod
+    def __command_match_unset(self, val: str) -> bool:
+        raise NotImplementedError("Hmm")
+
+    async def __call__(
+        self,
+        evt: MaubotMessageEvent,
+        *,
+        _existing_args: Dict[str, Any] = None,
+        remaining_val: str = None,
+    ) -> Any:
+        if evt.sender == evt.client.mxid or evt.content.msgtype not in self.__mb_msgtypes__:
+            return
+        if remaining_val is None:
+            if not evt.content.body or evt.content.body[0] != "!":
+                return
+            command, remaining_val = _split_in_two(evt.content.body[1:], " ")
+            command = command.lower()
+            if not self.__mb_is_command_match__(self.__bound_instance__, command):
+                return
+        call_args: Dict[str, Any] = {**_existing_args} if _existing_args else {}
+
+        if not self.__mb_arg_fallthrough__ and len(self.__mb_subcommands__) > 0:
+            ok, res = await self.__call_subcommand__(evt, call_args, remaining_val)
+            if ok:
+                return res
+
+        ok, remaining_val = await self.__parse_args__(evt, call_args, remaining_val)
+        if not ok:
+            return
+        elif self.__mb_arg_fallthrough__ and len(self.__mb_subcommands__) > 0:
+            ok, res = await self.__call_subcommand__(evt, call_args, remaining_val)
+            if ok:
+                return res
+            elif self.__mb_require_subcommand__:
+                await evt.reply(self.__mb_full_help__)
+                return
+
+        if self.__mb_must_consume_args__ and remaining_val.strip():
+            await evt.reply(self.__mb_full_help__)
+            return
+
+        if self.__bound_instance__:
+            return await self.__mb_func__(self.__bound_instance__, evt, **call_args)
+        return await self.__mb_func__(evt, **call_args)
+
+    async def __call_subcommand__(
+        self, evt: MaubotMessageEvent, call_args: Dict[str, Any], remaining_val: str
+    ) -> Tuple[bool, Any]:
+        command, remaining_val = _split_in_two(remaining_val.strip(), " ")
+        for subcommand in self.__mb_subcommands__:
+            if subcommand.__mb_is_command_match__(subcommand.__bound_instance__, command):
+                return True, await subcommand(
+                    evt, _existing_args=call_args, remaining_val=remaining_val
+                )
+        return False, None
+
+    async def __parse_args__(
+        self, evt: MaubotMessageEvent, call_args: Dict[str, Any], remaining_val: str
+    ) -> Tuple[bool, str]:
+        for arg in self.__mb_arguments__:
+            try:
+                remaining_val, call_args[arg.name] = arg.match(
+                    remaining_val.strip(), evt=evt, instance=self.__bound_instance__
+                )
+                if arg.required and call_args[arg.name] is None:
+                    raise ValueError("Argument required")
+            except ArgumentSyntaxError as e:
+                await evt.reply(e.message + (f"\n{self.__mb_usage__}" if e.show_usage else ""))
+                return False, remaining_val
+            except ValueError:
+                await evt.reply(self.__mb_usage__)
+                return False, remaining_val
+        return True, remaining_val
+
+    @property
+    def __mb_full_help__(self) -> str:
+        usage = self.__mb_usage_without_subcommands__ + "\n\n"
+        if not self.__mb_require_subcommand__:
+            usage += f"* {self.__mb_prefix__} {self.__mb_usage_args__} - {self.__mb_help__}\n"
+        usage += "\n".join(cmd.__mb_usage_inline__ for cmd in self.__mb_subcommands__)
+        return usage
+
+    @property
+    def __mb_usage_args__(self) -> str:
+        arg_usage = " ".join(
+            f"<{arg.label}>" if arg.required else f"[{arg.label}]" for arg in self.__mb_arguments__
+        )
+        if self.__mb_subcommands__ and self.__mb_arg_fallthrough__:
+            arg_usage += " " + self.__mb_usage_subcommand__
+        return arg_usage
+
+    @property
+    def __mb_usage_subcommand__(self) -> str:
+        return f"<subcommand> [...]"
+
+    @property
+    def __mb_name__(self) -> str:
+        return self.__mb_get_name__(self.__bound_instance__)
+
+    @property
+    def __mb_prefix__(self) -> str:
+        if self.__mb_parent__:
+            return (
+                f"!{self.__mb_parent__.__mb_get_name__(self.__bound_instance__)} "
+                f"{self.__mb_name__}"
+            )
+        return f"!{self.__mb_name__}"
+
+    @property
+    def __mb_usage_inline__(self) -> str:
+        if not self.__mb_arg_fallthrough__:
+            return (
+                f"* {self.__mb_name__} {self.__mb_usage_args__} - {self.__mb_help__}\n"
+                f"* {self.__mb_name__} {self.__mb_usage_subcommand__}"
+            )
+        return f"* {self.__mb_name__} {self.__mb_usage_args__} - {self.__mb_help__}"
+
+    @property
+    def __mb_subcommands_list__(self) -> str:
+        return f"**Subcommands:** {', '.join(sc.__mb_name__ for sc in self.__mb_subcommands__)}"
+
+    @property
+    def __mb_usage_without_subcommands__(self) -> str:
+        if not self.__mb_arg_fallthrough__:
+            if not self.__mb_arguments__:
+                return f"**Usage:** {self.__mb_prefix__} [subcommand] [...]"
+            return (
+                f"**Usage:** {self.__mb_prefix__} {self.__mb_usage_args__}"
+                f" _OR_ {self.__mb_prefix__} {self.__mb_usage_subcommand__}"
+            )
+        return f"**Usage:** {self.__mb_prefix__} {self.__mb_usage_args__}"
+
+    @property
+    def __mb_usage__(self) -> str:
+        if len(self.__mb_subcommands__) > 0:
+            return f"{self.__mb_usage_without_subcommands__}  \n{self.__mb_subcommands_list__}"
+        return self.__mb_usage_without_subcommands__
+
+    def subcommand(
+        self,
+        name: PrefixType = None,
+        *,
+        help: str = None,
+        aliases: AliasesType = None,
+        required_subcommand: bool = True,
+        arg_fallthrough: bool = True,
+    ) -> CommandHandlerDecorator:
+        def decorator(func: Union[CommandHandler, CommandHandlerFunc]) -> CommandHandler:
+            if not isinstance(func, CommandHandler):
+                func = CommandHandler(func)
+            new(
+                name,
+                help=help,
+                aliases=aliases,
+                require_subcommand=required_subcommand,
+                arg_fallthrough=arg_fallthrough,
+            )(func)
+            func.__mb_parent__ = self
+            func.__mb_event_handler__ = False
+            self.__mb_subcommands__.append(func)
+            return func
+
+        return decorator
+
+
+def new(
+    name: PrefixType = None,
+    *,
+    help: str = None,
+    aliases: AliasesType = None,
+    event_type: EventType = EventType.ROOM_MESSAGE,
+    msgtypes: Iterable[MessageType] = None,
+    require_subcommand: bool = True,
+    arg_fallthrough: bool = True,
+    must_consume_args: bool = True,
+) -> CommandHandlerDecorator:
+    def decorator(func: Union[CommandHandler, CommandHandlerFunc]) -> CommandHandler:
+        if not isinstance(func, CommandHandler):
+            func = CommandHandler(func)
+        func.__mb_help__ = help
+        if name:
+            if callable(name):
+                if len(inspect.getfullargspec(name).args) == 0:
+                    func.__mb_get_name__ = lambda self: name()
+                else:
+                    func.__mb_get_name__ = name
+            else:
+                func.__mb_get_name__ = lambda self: name
+        else:
+            func.__mb_get_name__ = lambda self: func.__mb_func__.__name__.replace("_", "-")
+        if callable(aliases):
+            if len(inspect.getfullargspec(aliases).args) == 1:
+                func.__mb_is_command_match__ = lambda self, val: aliases(val)
+            else:
+                func.__mb_is_command_match__ = aliases
+        elif isinstance(aliases, (list, set, tuple)):
+            func.__mb_is_command_match__ = lambda self, val: (
+                val == func.__mb_get_name__(self) or val in aliases
+            )
+        else:
+            func.__mb_is_command_match__ = lambda self, val: val == func.__mb_get_name__(self)
+        # Decorators are executed last to first, so we reverse the argument list.
+        func.__mb_arguments__.reverse()
+        func.__mb_require_subcommand__ = require_subcommand
+        func.__mb_arg_fallthrough__ = arg_fallthrough
+        func.__mb_must_consume_args__ = must_consume_args
+        func.__mb_event_types__ = {event_type}
+        if msgtypes:
+            func.__mb_msgtypes__ = msgtypes
+        return func
+
+    return decorator
+
+
+class ArgumentSyntaxError(ValueError):
+    def __init__(self, message: str, show_usage: bool = True) -> None:
+        super().__init__(message)
+        self.message = message
+        self.show_usage = show_usage
+
+
+class Argument(ABC):
+    def __init__(
+        self, name: str, label: str = None, *, required: bool = False, pass_raw: bool = False
+    ) -> None:
+        self.name = name
+        self.label = label or name
+        self.required = required
+        self.pass_raw = pass_raw
+
+    @abstractmethod
+    def match(self, val: str, **kwargs) -> Tuple[str, Any]:
+        pass
+
+    def __call__(self, func: Union[CommandHandler, CommandHandlerFunc]) -> CommandHandler:
+        if not isinstance(func, CommandHandler):
+            func = CommandHandler(func)
+        func.__mb_arguments__.append(self)
+        return func
+
+
+class RegexArgument(Argument):
+    def __init__(
+        self,
+        name: str,
+        label: str = None,
+        *,
+        required: bool = False,
+        pass_raw: bool = False,
+        matches: str = None,
+    ) -> None:
+        super().__init__(name, label, required=required, pass_raw=pass_raw)
+        matches = f"^{matches}" if self.pass_raw else f"^{matches}$"
+        self.regex = re.compile(matches)
+
+    def match(self, val: str, **kwargs) -> Tuple[str, Any]:
+        orig_val = val
+        if not self.pass_raw:
+            val = re.split(r"\s", val, 1)[0]
+        match = self.regex.match(val)
+        if match:
+            return (
+                orig_val[: match.start()] + orig_val[match.end() :],
+                match.groups() or val[match.start() : match.end()],
+            )
+        return orig_val, None
+
+
+class CustomArgument(Argument):
+    def __init__(
+        self,
+        name: str,
+        label: str = None,
+        *,
+        required: bool = False,
+        pass_raw: bool = False,
+        matcher: Callable[[str], Any],
+    ) -> None:
+        super().__init__(name, label, required=required, pass_raw=pass_raw)
+        self.matcher = matcher
+
+    def match(self, val: str, **kwargs) -> Tuple[str, Any]:
+        if self.pass_raw:
+            return self.matcher(val)
+        orig_val = val
+        val = re.split(r"\s", val, 1)[0]
+        res = self.matcher(val)
+        if res is not None:
+            return orig_val[len(val) :], res
+        return orig_val, None
+
+
+class SimpleArgument(Argument):
+    def match(self, val: str, **kwargs) -> Tuple[str, Any]:
+        if self.pass_raw:
+            return "", val
+        res = re.split(r"\s", val, 1)[0]
+        return val[len(res) :], res
+
+
+def argument(
+    name: str,
+    label: str = None,
+    *,
+    required: bool = True,
+    matches: Optional[str] = None,
+    parser: Optional[Callable[[str], Any]] = None,
+    pass_raw: bool = False,
+) -> CommandHandlerDecorator:
+    if matches:
+        return RegexArgument(name, label, required=required, matches=matches, pass_raw=pass_raw)
+    elif parser:
+        return CustomArgument(name, label, required=required, matcher=parser, pass_raw=pass_raw)
+    else:
+        return SimpleArgument(name, label, required=required, pass_raw=pass_raw)
+
+
+def passive(
+    regex: Union[str, Pattern],
+    *,
+    msgtypes: Sequence[MessageType] = (MessageType.TEXT,),
+    field: Callable[[MaubotMessageEvent], str] = lambda evt: evt.content.body,
+    event_type: EventType = EventType.ROOM_MESSAGE,
+    multiple: bool = False,
+    case_insensitive: bool = False,
+    multiline: bool = False,
+    dot_all: bool = False,
+) -> PassiveCommandHandlerDecorator:
+    if not isinstance(regex, Pattern):
+        flags = re.RegexFlag.UNICODE
+        if case_insensitive:
+            flags |= re.IGNORECASE
+        if multiline:
+            flags |= re.MULTILINE
+        if dot_all:
+            flags |= re.DOTALL
+        regex = re.compile(regex, flags=flags)
+
+    def decorator(func: CommandHandlerFunc) -> CommandHandlerFunc:
+        combine = None
+        if hasattr(func, "__mb_passive_orig__"):
+            combine = func
+            func = func.__mb_passive_orig__
+
+        @event.on(event_type)
+        @functools.wraps(func)
+        async def replacement(self, evt: MaubotMessageEvent = None) -> None:
+            if not evt and isinstance(self, MaubotMessageEvent):
+                evt = self
+                self = None
+            if evt.sender == evt.client.mxid:
+                return
+            elif msgtypes and evt.content.msgtype not in msgtypes:
+                return
+            data = field(evt)
+            if multiple:
+                val = [
+                    (data[match.pos : match.endpos], *match.groups())
+                    for match in regex.finditer(data)
+                ]
+            else:
+                match = regex.search(data)
+                if match:
+                    val = (data[match.pos : match.endpos], *match.groups())
+                else:
+                    val = None
+            if val:
+                if self:
+                    await func(self, evt, val)
+                else:
+                    await func(evt, val)
+
+        if combine:
+            orig_replacement = replacement
+
+            @event.on(event_type)
+            @functools.wraps(func)
+            async def replacement(self, evt: MaubotMessageEvent = None) -> None:
+                await asyncio.gather(combine(self, evt), orig_replacement(self, evt))
+
+        replacement.__mb_passive_orig__ = func
+
+        return replacement
+
+    return decorator

maubot-src/maubot/handlers/event.py

@@ -0,0 +1,44 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Callable, NewType
+
+from mautrix.client import EventHandler, InternalEventType
+from mautrix.types import EventType
+
+EventHandlerDecorator = NewType("EventHandlerDecorator", Callable[[EventHandler], EventHandler])
+
+
+def on(var: EventType | InternalEventType | EventHandler) -> EventHandlerDecorator | EventHandler:
+    def decorator(func: EventHandler) -> EventHandler:
+        func.__mb_event_handler__ = True
+        if isinstance(var, (EventType, InternalEventType)):
+            if hasattr(func, "__mb_event_types__"):
+                func.__mb_event_types__.add(var)
+            else:
+                func.__mb_event_types__ = {var}
+        else:
+            func.__mb_event_types__ = {EventType.ALL}
+
+        return func
+
+    return decorator if isinstance(var, (EventType, InternalEventType)) else decorator(var)
+
+
+def off(func: EventHandler) -> EventHandler:
+    func.__mb_event_handler__ = False
+    return func

maubot-src/maubot/handlers/web.py

@@ -0,0 +1,66 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Awaitable, Callable
+
+from aiohttp import hdrs, web
+
+WebHandler = Callable[[web.Request], Awaitable[web.StreamResponse]]
+WebHandlerDecorator = Callable[[WebHandler], WebHandler]
+
+
+def head(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_HEAD, path, **kwargs)
+
+
+def options(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_OPTIONS, path, **kwargs)
+
+
+def get(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_GET, path, **kwargs)
+
+
+def post(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_POST, path, **kwargs)
+
+
+def put(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_PUT, path, **kwargs)
+
+
+def patch(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_PATCH, path, **kwargs)
+
+
+def delete(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_DELETE, path, **kwargs)
+
+
+def view(path: str, **kwargs: Any) -> WebHandlerDecorator:
+    return handle(hdrs.METH_ANY, path, **kwargs)
+
+
+def handle(method: str, path: str, **kwargs) -> WebHandlerDecorator:
+    def decorator(handler: WebHandler) -> WebHandler:
+        try:
+            handlers = getattr(handler, "__mb_web_handler__")
+        except AttributeError:
+            handlers = []
+            setattr(handler, "__mb_web_handler__", handlers)
+        handlers.append((method, path, kwargs))
+        return handler
+
+    return decorator

maubot-src/maubot/instance.py

@@ -0,0 +1,533 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, AsyncGenerator, cast
+from collections import defaultdict
+import asyncio
+import inspect
+import io
+import logging
+import os.path
+
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+
+from mautrix.types import UserID
+from mautrix.util import background_task
+from mautrix.util.async_db import Database, Scheme, UpgradeTable
+from mautrix.util.async_getter_lock import async_getter_lock
+from mautrix.util.config import BaseProxyConfig, RecursiveDict
+from mautrix.util.logging import TraceLogger
+
+from .client import Client
+from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
+from .lib.plugin_db import ProxyPostgresDatabase
+from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
+from .plugin_base import Plugin
+
+if TYPE_CHECKING:
+    from .__main__ import Maubot
+    from .server import PluginWebApp
+
+log: TraceLogger = cast(TraceLogger, logging.getLogger("maubot.instance"))
+db_log: TraceLogger = cast(TraceLogger, logging.getLogger("maubot.instance_db"))
+
+yaml = YAML()
+yaml.indent(4)
+yaml.width = 200
+
+
+class PluginInstance(DBInstance):
+    maubot: "Maubot" = None
+    cache: dict[str, PluginInstance] = {}
+    plugin_directories: list[str] = []
+    _async_get_locks: dict[Any, asyncio.Lock] = defaultdict(lambda: asyncio.Lock())
+
+    log: logging.Logger
+    loader: PluginLoader | None
+    client: Client | None
+    plugin: Plugin | None
+    config: BaseProxyConfig | None
+    base_cfg: RecursiveDict[CommentedMap] | None
+    base_cfg_str: str | None
+    inst_db: sql.engine.Engine | Database | None
+    inst_db_tables: dict | None
+    inst_webapp: PluginWebApp | None
+    inst_webapp_url: str | None
+    started: bool
+
+    def __init__(
+        self,
+        id: str,
+        type: str,
+        enabled: bool,
+        primary_user: UserID,
+        config: str = "",
+        database_engine: DatabaseEngine | None = None,
+    ) -> None:
+        super().__init__(
+            id=id,
+            type=type,
+            enabled=bool(enabled),
+            primary_user=primary_user,
+            config_str=config,
+            database_engine=database_engine,
+        )
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    @classmethod
+    def init_cls(cls, maubot: "Maubot") -> None:
+        cls.maubot = maubot
+
+    def postinit(self) -> None:
+        self.log = log.getChild(self.id)
+        self.cache[self.id] = self
+        self.config = None
+        self.started = False
+        self.loader = None
+        self.client = None
+        self.plugin = None
+        self.inst_db = None
+        self.inst_db_tables = None
+        self.inst_webapp = None
+        self.inst_webapp_url = None
+        self.base_cfg = None
+        self.base_cfg_str = None
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "type": self.type,
+            "enabled": self.enabled,
+            "started": self.started,
+            "primary_user": self.primary_user,
+            "config": self.config_str,
+            "base_config": self.base_cfg_str,
+            "database": (
+                self.inst_db is not None and self.maubot.config["api_features.instance_database"]
+            ),
+            "database_interface": self.loader.meta.database_type_str if self.loader else "unknown",
+            "database_engine": self.database_engine_str,
+        }
+
+    def _introspect_sqlalchemy(self) -> dict:
+        metadata = MetaData()
+        metadata.reflect(self.inst_db)
+        return {
+            table.name: {
+                "columns": {
+                    column.name: {
+                        "type": str(column.type),
+                        "unique": column.unique or False,
+                        "default": column.default,
+                        "nullable": column.nullable,
+                        "primary": column.primary_key,
+                    }
+                    for column in table.columns
+                },
+            }
+            for table in metadata.tables.values()
+        }
+
+    async def _introspect_sqlite(self) -> dict:
+        q = """
+        SELECT
+            m.name AS table_name,
+            p.cid AS col_id,
+            p.name AS column_name,
+            p.type AS data_type,
+            p.pk AS is_primary,
+            p.dflt_value AS column_default,
+            p.[notnull] AS is_nullable
+        FROM sqlite_master m
+        LEFT JOIN pragma_table_info((m.name)) p
+        WHERE m.type = 'table'
+        ORDER BY table_name, col_id
+        """
+        data = await self.inst_db.fetch(q)
+        tables = defaultdict(lambda: {"columns": {}})
+        for column in data:
+            table_name = column["table_name"]
+            col_name = column["column_name"]
+            tables[table_name]["columns"][col_name] = {
+                "type": column["data_type"],
+                "nullable": bool(column["is_nullable"]),
+                "default": column["column_default"],
+                "primary": bool(column["is_primary"]),
+                # TODO uniqueness?
+            }
+        return tables
+
+    async def _introspect_postgres(self) -> dict:
+        assert isinstance(self.inst_db, ProxyPostgresDatabase)
+        q = """
+        SELECT col.table_name, col.column_name, col.data_type, col.is_nullable, col.column_default,
+               tc.constraint_type
+        FROM information_schema.columns col
+        LEFT JOIN information_schema.constraint_column_usage ccu
+               ON ccu.column_name=col.column_name
+        LEFT JOIN information_schema.table_constraints tc
+               ON col.table_name=tc.table_name
+              AND col.table_schema=tc.table_schema
+              AND ccu.constraint_name=tc.constraint_name
+              AND ccu.constraint_schema=tc.constraint_schema
+              AND tc.constraint_type IN ('PRIMARY KEY', 'UNIQUE')
+        WHERE col.table_schema=$1
+        """
+        data = await self.inst_db.fetch(q, self.inst_db.schema_name)
+        tables = defaultdict(lambda: {"columns": {}})
+        for column in data:
+            table_name = column["table_name"]
+            col_name = column["column_name"]
+            tables[table_name]["columns"].setdefault(
+                col_name,
+                {
+                    "type": column["data_type"],
+                    "nullable": column["is_nullable"],
+                    "default": column["column_default"],
+                    "primary": False,
+                    "unique": False,
+                },
+            )
+            if column["constraint_type"] == "PRIMARY KEY":
+                tables[table_name]["columns"][col_name]["primary"] = True
+            elif column["constraint_type"] == "UNIQUE":
+                tables[table_name]["columns"][col_name]["unique"] = True
+        return tables
+
+    async def get_db_tables(self) -> dict:
+        if self.inst_db_tables is None:
+            if isinstance(self.inst_db, Engine):
+                self.inst_db_tables = self._introspect_sqlalchemy()
+            elif self.inst_db.scheme == Scheme.SQLITE:
+                self.inst_db_tables = await self._introspect_sqlite()
+            else:
+                self.inst_db_tables = await self._introspect_postgres()
+        return self.inst_db_tables
+
+    async def load(self) -> bool:
+        if not self.loader:
+            try:
+                self.loader = PluginLoader.find(self.type)
+            except KeyError:
+                self.log.error(f"Failed to find loader for type {self.type}")
+                await self.update_enabled(False)
+                return False
+        if not self.client:
+            self.client = await Client.get(self.primary_user)
+            if not self.client:
+                self.log.error(f"Failed to get client for user {self.primary_user}")
+                await self.update_enabled(False)
+                return False
+        if self.loader.meta.webapp:
+            self.enable_webapp()
+        self.log.debug("Plugin instance dependencies loaded")
+        self.loader.references.add(self)
+        self.client.references.add(self)
+        return True
+
+    def enable_webapp(self) -> None:
+        self.inst_webapp, self.inst_webapp_url = self.maubot.server.get_instance_subapp(self.id)
+
+    def disable_webapp(self) -> None:
+        self.maubot.server.remove_instance_webapp(self.id)
+        self.inst_webapp = None
+        self.inst_webapp_url = None
+
+    @property
+    def _sqlite_db_path(self) -> str:
+        return os.path.join(self.maubot.config["plugin_databases.sqlite"], f"{self.id}.db")
+
+    async def delete(self) -> None:
+        if self.loader is not None:
+            self.loader.references.remove(self)
+        if self.client is not None:
+            self.client.references.remove(self)
+        try:
+            del self.cache[self.id]
+        except KeyError:
+            pass
+        await super().delete()
+        if self.inst_db:
+            await self.stop_database()
+            await self.delete_database()
+        if self.inst_webapp:
+            self.disable_webapp()
+
+    def load_config(self) -> CommentedMap:
+        return yaml.load(self.config_str)
+
+    def save_config(self, data: RecursiveDict[CommentedMap]) -> None:
+        buf = io.StringIO()
+        yaml.dump(data, buf)
+        val = buf.getvalue()
+        if val != self.config_str:
+            self.config_str = val
+            self.log.debug("Creating background task to save updated config")
+            background_task.create(self.update())
+
+    async def start_database(
+        self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True
+    ) -> None:
+        if self.loader.meta.database_type == DatabaseType.SQLALCHEMY:
+            if self.database_engine is None:
+                await self.update_db_engine(DatabaseEngine.SQLITE)
+            elif self.database_engine == DatabaseEngine.POSTGRES:
+                raise RuntimeError(
+                    "Instance database engine is marked as Postgres, but plugin uses legacy "
+                    "database interface, which doesn't support postgres."
+                )
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
+        elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
+            if self.database_engine is None:
+                if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
+                    await self.update_db_engine(DatabaseEngine.SQLITE)
+                else:
+                    await self.update_db_engine(DatabaseEngine.POSTGRES)
+            instance_db_log = db_log.getChild(self.id)
+            if self.database_engine == DatabaseEngine.POSTGRES:
+                if not self.maubot.plugin_postgres_db:
+                    raise RuntimeError(
+                        "Instance database engine is marked as Postgres, but this maubot isn't "
+                        "configured to support Postgres for plugin databases"
+                    )
+                self.inst_db = ProxyPostgresDatabase(
+                    pool=self.maubot.plugin_postgres_db,
+                    instance_id=self.id,
+                    max_conns=self.maubot.config["plugin_databases.postgres_max_conns_per_plugin"],
+                    upgrade_table=upgrade_table,
+                    log=instance_db_log,
+                )
+            else:
+                self.inst_db = Database.create(
+                    f"sqlite:{self._sqlite_db_path}",
+                    upgrade_table=upgrade_table,
+                    log=instance_db_log,
+                )
+            if actually_start:
+                await self.inst_db.start()
+        else:
+            raise RuntimeError(f"Unrecognized database type {self.loader.meta.database_type}")
+
+    async def stop_database(self) -> None:
+        if isinstance(self.inst_db, Database):
+            await self.inst_db.stop()
+        elif isinstance(self.inst_db, Engine):
+            self.inst_db.dispose()
+        else:
+            raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")
+
+    async def delete_database(self) -> None:
+        if self.loader.meta.database_type == DatabaseType.SQLALCHEMY:
+            ZippedPluginLoader.trash(self._sqlite_db_path, reason="deleted")
+        elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
+            if self.inst_db is None:
+                await self.start_database(None, actually_start=False)
+            if isinstance(self.inst_db, ProxyPostgresDatabase):
+                await self.inst_db.delete()
+            else:
+                ZippedPluginLoader.trash(self._sqlite_db_path, reason="deleted")
+        else:
+            raise RuntimeError(f"Unrecognized database type {self.loader.meta.database_type}")
+        self.inst_db = None
+
+    async def start(self) -> None:
+        if self.started:
+            self.log.warning("Ignoring start() call to already started plugin")
+            return
+        elif not self.enabled:
+            self.log.warning("Plugin disabled, not starting.")
+            return
+        if not self.client or not self.loader:
+            self.log.warning("Missing plugin instance dependencies, attempting to load...")
+            if not await self.load():
+                return
+        cls = await self.loader.load()
+        if self.loader.meta.webapp and self.inst_webapp is None:
+            self.log.debug("Enabling webapp after plugin meta reload")
+            self.enable_webapp()
+        elif not self.loader.meta.webapp and self.inst_webapp is not None:
+            self.log.debug("Disabling webapp after plugin meta reload")
+            self.disable_webapp()
+        if self.loader.meta.database:
+            try:
+                await self.start_database(cls.get_db_upgrade_table())
+            except Exception:
+                self.log.exception("Failed to start instance database")
+                await self.update_enabled(False)
+                return
+        config_class = cls.get_config_class()
+        if config_class:
+            try:
+                base = await self.loader.read_file("base-config.yaml")
+                self.base_cfg = RecursiveDict(yaml.load(base.decode("utf-8")), CommentedMap)
+                buf = io.StringIO()
+                yaml.dump(self.base_cfg._data, buf)
+                self.base_cfg_str = buf.getvalue()
+            except (FileNotFoundError, KeyError):
+                self.base_cfg = None
+                self.base_cfg_str = None
+            if self.base_cfg:
+                base_cfg_func = self.base_cfg.clone
+            else:
+
+                def base_cfg_func() -> None:
+                    return None
+
+            self.config = config_class(self.load_config, base_cfg_func, self.save_config)
+        self.plugin = cls(
+            client=self.client.client,
+            loop=self.maubot.loop,
+            http=self.client.http_client,
+            instance_id=self.id,
+            log=self.log,
+            config=self.config,
+            database=self.inst_db,
+            loader=self.loader,
+            webapp=self.inst_webapp,
+            webapp_url=self.inst_webapp_url,
+        )
+        try:
+            await self.plugin.internal_start()
+        except Exception:
+            self.log.exception("Failed to start instance")
+            await self.update_enabled(False)
+            return
+        self.started = True
+        self.inst_db_tables = None
+        self.log.info(
+            f"Started instance of {self.loader.meta.id} v{self.loader.meta.version} "
+            f"with user {self.client.id}"
+        )
+
+    async def stop(self) -> None:
+        if not self.started:
+            self.log.warning("Ignoring stop() call to non-running plugin")
+            return
+        self.log.debug("Stopping plugin instance...")
+        self.started = False
+        try:
+            await self.plugin.internal_stop()
+        except Exception:
+            self.log.exception("Failed to stop instance")
+        self.plugin = None
+        if self.inst_db:
+            try:
+                await self.stop_database()
+            except Exception:
+                self.log.exception("Failed to stop instance database")
+        self.inst_db_tables = None
+
+    async def update_id(self, new_id: str | None) -> None:
+        if new_id is not None and new_id.lower() != self.id:
+            await super().update_id(new_id.lower())
+
+    async def update_config(self, config: str | None) -> None:
+        if config is None or self.config_str == config:
+            return
+        self.config_str = config
+        if self.started and self.plugin is not None:
+            res = self.plugin.on_external_config_update()
+            if inspect.isawaitable(res):
+                await res
+        await self.update()
+
+    async def update_primary_user(self, primary_user: UserID | None) -> bool:
+        if primary_user is None or primary_user == self.primary_user:
+            return True
+        client = await Client.get(primary_user)
+        if not client:
+            return False
+        await self.stop()
+        self.primary_user = client.id
+        if self.client:
+            self.client.references.remove(self)
+        self.client = client
+        self.client.references.add(self)
+        await self.update()
+        await self.start()
+        self.log.debug(f"Primary user switched to {self.client.id}")
+        return True
+
+    async def update_type(self, type: str | None) -> bool:
+        if type is None or type == self.type:
+            return True
+        try:
+            loader = PluginLoader.find(type)
+        except KeyError:
+            return False
+        await self.stop()
+        self.type = loader.meta.id
+        if self.loader:
+            self.loader.references.remove(self)
+        self.loader = loader
+        self.loader.references.add(self)
+        await self.update()
+        await self.start()
+        self.log.debug(f"Type switched to {self.loader.meta.id}")
+        return True
+
+    async def update_started(self, started: bool) -> None:
+        if started is not None and started != self.started:
+            await (self.start() if started else self.stop())
+
+    async def update_enabled(self, enabled: bool) -> None:
+        if enabled is not None and enabled != self.enabled:
+            self.enabled = enabled
+            await self.update()
+
+    async def update_db_engine(self, db_engine: DatabaseEngine | None) -> None:
+        if db_engine is not None and db_engine != self.database_engine:
+            self.database_engine = db_engine
+            await self.update()
+
+    @classmethod
+    @async_getter_lock
+    async def get(
+        cls, instance_id: str, *, type: str | None = None, primary_user: UserID | None = None
+    ) -> PluginInstance | None:
+        try:
+            return cls.cache[instance_id]
+        except KeyError:
+            pass
+
+        instance = cast(cls, await super().get(instance_id))
+        if instance is not None:
+            instance.postinit()
+            return instance
+
+        if type and primary_user:
+            instance = cls(instance_id, type=type, enabled=True, primary_user=primary_user)
+            await instance.insert()
+            instance.postinit()
+            return instance
+
+        return None
+
+    @classmethod
+    async def all(cls) -> AsyncGenerator[PluginInstance, None]:
+        instances = await super().all()
+        instance: PluginInstance
+        for instance in instances:
+            try:
+                yield cls.cache[instance.id]
+            except KeyError:
+                instance.postinit()
+                yield instance

maubot-src/maubot/lib/color_log.py

@@ -0,0 +1,49 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.logging.color import (
+    MAU_COLOR,
+    MXID_COLOR,
+    PREFIX,
+    RESET,
+    ColorFormatter as BaseColorFormatter,
+)
+
+INST_COLOR = PREFIX + "35m"  # magenta
+LOADER_COLOR = PREFIX + "36m"  # blue
+
+
+class ColorFormatter(BaseColorFormatter):
+    def _color_name(self, module: str) -> str:
+        client = "maubot.client"
+        if module.startswith(client + "."):
+            suffix = ""
+            if module.endswith(".crypto"):
+                suffix = f".{MAU_COLOR}crypto{RESET}"
+                module = module[: -len(".crypto")]
+            module = module[len(client) + 1 :]
+            return f"{MAU_COLOR}{client}{RESET}.{MXID_COLOR}{module}{RESET}{suffix}"
+        instance = "maubot.instance"
+        if module.startswith(instance + "."):
+            return f"{MAU_COLOR}{instance}{RESET}.{INST_COLOR}{module[len(instance) + 1:]}{RESET}"
+        instance_db = "maubot.instance_db"
+        if module.startswith(instance_db + "."):
+            return f"{MAU_COLOR}{instance_db}{RESET}.{INST_COLOR}{module[len(instance_db) + 1:]}{RESET}"
+        loader = "maubot.loader"
+        if module.startswith(loader + "."):
+            return f"{MAU_COLOR}{instance}{RESET}.{LOADER_COLOR}{module[len(loader) + 1:]}{RESET}"
+        if module.startswith("maubot."):
+            return f"{MAU_COLOR}{module}{RESET}"
+        return super()._color_name(module)

maubot-src/maubot/lib/future_awaitable.py

@@ -0,0 +1,9 @@
+from typing import Any, Awaitable, Callable, Generator
+
+
+class FutureAwaitable:
+    def __init__(self, func: Callable[[], Awaitable[None]]) -> None:
+        self._func = func
+
+    def __await__(self) -> Generator[Any, None, None]:
+        return self._func().__await__()

maubot-src/maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot-src/maubot/lib/plugin_db.py

@@ -0,0 +1,100 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+import asyncio
+
+from mautrix.util.async_db import Database, PostgresDatabase, Scheme, UpgradeTable
+from mautrix.util.async_db.connection import LoggingConnection
+from mautrix.util.logging import TraceLogger
+
+remove_double_quotes = str.maketrans({'"': "_"})
+
+
+class ProxyPostgresDatabase(Database):
+    scheme = Scheme.POSTGRES
+    _underlying_pool: PostgresDatabase
+    schema_name: str
+    _quoted_schema: str
+    _default_search_path: str
+    _conn_sema: asyncio.Semaphore
+    _max_conns: int
+
+    def __init__(
+        self,
+        pool: PostgresDatabase,
+        instance_id: str,
+        max_conns: int,
+        upgrade_table: UpgradeTable | None,
+        log: TraceLogger | None = None,
+    ) -> None:
+        super().__init__(pool.url, upgrade_table=upgrade_table, log=log)
+        self._underlying_pool = pool
+        # Simple accidental SQL injection prevention.
+        # Doesn't have to be perfect, since plugin instance IDs can only be set by admins anyway.
+        self.schema_name = f"mbp_{instance_id.translate(remove_double_quotes)}"
+        self._quoted_schema = f'"{self.schema_name}"'
+        self._default_search_path = '"$user", public'
+        self._conn_sema = asyncio.BoundedSemaphore(max_conns)
+        self._max_conns = max_conns
+
+    async def start(self) -> None:
+        async with self._underlying_pool.acquire_direct() as conn:
+            self._default_search_path = await conn.fetchval("SHOW search_path")
+            self.log.trace(f"Found default search path: {self._default_search_path}")
+            await conn.execute(f"CREATE SCHEMA IF NOT EXISTS {self._quoted_schema}")
+        await super().start()
+
+    async def stop(self) -> None:
+        for _ in range(self._max_conns):
+            try:
+                await asyncio.wait_for(self._conn_sema.acquire(), timeout=3)
+            except asyncio.TimeoutError:
+                self.log.warning(
+                    "Failed to drain plugin database connection pool, "
+                    "the plugin may be leaking database connections"
+                )
+                break
+
+    async def delete(self) -> None:
+        self.log.info(f"Deleting schema {self.schema_name} and all data in it")
+        try:
+            await self._underlying_pool.execute(
+                f"DROP SCHEMA IF EXISTS {self._quoted_schema} CASCADE"
+            )
+        except Exception:
+            self.log.warning("Failed to delete schema", exc_info=True)
+
+    @asynccontextmanager
+    async def acquire_direct(self) -> LoggingConnection:
+        conn: LoggingConnection
+        async with self._conn_sema, self._underlying_pool.acquire_direct() as conn:
+            await conn.execute(f"SET search_path = {self._quoted_schema}")
+            try:
+                yield conn
+            finally:
+                if not conn.wrapped.is_closed():
+                    try:
+                        await conn.execute(f"SET search_path = {self._default_search_path}")
+                    except Exception:
+                        self.log.exception("Error resetting search_path after use")
+                        await conn.wrapped.close()
+                else:
+                    self.log.debug("Connection was closed after use, not resetting search_path")
+
+
+__all__ = ["ProxyPostgresDatabase"]

maubot-src/maubot/lib/state_store.py

@@ -0,0 +1,27 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.client.state_store.asyncpg import PgStateStore as BasePgStateStore
+
+try:
+    from mautrix.crypto import StateStore as CryptoStateStore
+
+    class PgStateStore(BasePgStateStore, CryptoStateStore):
+        pass
+
+except ImportError as e:
+    PgStateStore = BasePgStateStore
+
+__all__ = ["PgStateStore"]

maubot-src/maubot/lib/zipimport.py

@@ -0,0 +1,794 @@
+# The pure Python implementation of zipimport in Python 3.8+. Slightly modified to allow clearing
+# the zip directory cache to bypass https://bugs.python.org/issue19081
+#
+# https://github.com/python/cpython/blob/5a5ce064b3baadcb79605c5a42ee3d0aee57cdfc/Lib/zipimport.py
+# See license at https://github.com/python/cpython/blob/master/LICENSE
+
+"""zipimport provides support for importing Python modules from Zip archives.
+
+This module exports three objects:
+- zipimporter: a class; its constructor takes a path to a Zip archive.
+- ZipImportError: exception raised by zipimporter objects. It's a
+  subclass of ImportError, so it can be caught as ImportError, too.
+- _zip_directory_cache: a dict, mapping archive paths to zip directory
+  info dicts, as used in zipimporter._files.
+
+It is usually not needed to use the zipimport module explicitly; it is
+used by the builtin import mechanism for sys.path items that are paths
+to Zip archives.
+"""
+
+from importlib import _bootstrap  # for _verbose_message
+from importlib import _bootstrap_external
+import marshal  # for loads
+import sys  # for modules
+import time  # for mktime
+
+import _imp  # for check_hash_based_pycs
+import _io  # for open
+
+__all__ = ["ZipImportError", "zipimporter"]
+
+
+def _unpack_uint32(data):
+    """Convert 4 bytes in little-endian to an integer."""
+    assert len(data) == 4
+    return int.from_bytes(data, "little")
+
+
+def _unpack_uint16(data):
+    """Convert 2 bytes in little-endian to an integer."""
+    assert len(data) == 2
+    return int.from_bytes(data, "little")
+
+
+path_sep = _bootstrap_external.path_sep
+alt_path_sep = _bootstrap_external.path_separators[1:]
+
+
+class ZipImportError(ImportError):
+    pass
+
+
+# _read_directory() cache
+_zip_directory_cache = {}
+
+_module_type = type(sys)
+
+END_CENTRAL_DIR_SIZE = 22
+STRING_END_ARCHIVE = b"PK\x05\x06"
+MAX_COMMENT_LEN = (1 << 16) - 1
+
+
+class zipimporter:
+    """zipimporter(archivepath) -> zipimporter object
+
+    Create a new zipimporter instance. 'archivepath' must be a path to
+    a zipfile, or to a specific path inside a zipfile. For example, it can be
+    '/tmp/myimport.zip', or '/tmp/myimport.zip/mydirectory', if mydirectory is a
+    valid directory inside the archive.
+
+    'ZipImportError is raised if 'archivepath' doesn't point to a valid Zip
+    archive.
+
+    The 'archive' attribute of zipimporter objects contains the name of the
+    zipfile targeted.
+    """
+
+    # Split the "subdirectory" from the Zip archive path, lookup a matching
+    # entry in sys.path_importer_cache, fetch the file directory from there
+    # if found, or else read it from the archive.
+    def __init__(self, path):
+        if not isinstance(path, str):
+            import os
+
+            path = os.fsdecode(path)
+        if not path:
+            raise ZipImportError("archive path is empty", path=path)
+        if alt_path_sep:
+            path = path.replace(alt_path_sep, path_sep)
+
+        prefix = []
+        while True:
+            try:
+                st = _bootstrap_external._path_stat(path)
+            except (OSError, ValueError):
+                # On Windows a ValueError is raised for too long paths.
+                # Back up one path element.
+                dirname, basename = _bootstrap_external._path_split(path)
+                if dirname == path:
+                    raise ZipImportError("not a Zip file", path=path)
+                path = dirname
+                prefix.append(basename)
+            else:
+                # it exists
+                if (st.st_mode & 0o170000) != 0o100000:  # stat.S_ISREG
+                    # it's a not file
+                    raise ZipImportError("not a Zip file", path=path)
+                break
+
+        try:
+            files = _zip_directory_cache[path]
+        except KeyError:
+            files = _read_directory(path)
+            _zip_directory_cache[path] = files
+        self._files = files
+        self.archive = path
+        # a prefix directory following the ZIP file path.
+        self.prefix = _bootstrap_external._path_join(*prefix[::-1])
+        if self.prefix:
+            self.prefix += path_sep
+
+    def reset_cache(self):
+        self._files = _read_directory(self.archive)
+        _zip_directory_cache[self.archive] = self._files
+
+    def remove_cache(self):
+        try:
+            del _zip_directory_cache[self.archive]
+        except KeyError:
+            pass
+
+    # Check whether we can satisfy the import of the module named by
+    # 'fullname', or whether it could be a portion of a namespace
+    # package. Return self if we can load it, a string containing the
+    # full path if it's a possible namespace portion, None if we
+    # can't load it.
+    def find_loader(self, fullname, path=None):
+        """find_loader(fullname, path=None) -> self, str or None.
+
+        Search for a module specified by 'fullname'. 'fullname' must be the
+        fully qualified (dotted) module name. It returns the zipimporter
+        instance itself if the module was found, a string containing the
+        full path name if it's possibly a portion of a namespace package,
+        or None otherwise. The optional 'path' argument is ignored -- it's
+        there for compatibility with the importer protocol.
+        """
+        mi = _get_module_info(self, fullname)
+        if mi is not None:
+            # This is a module or package.
+            return self, []
+
+        # Not a module or regular package. See if this is a directory, and
+        # therefore possibly a portion of a namespace package.
+
+        # We're only interested in the last path component of fullname
+        # earlier components are recorded in self.prefix.
+        modpath = _get_module_path(self, fullname)
+        if _is_dir(self, modpath):
+            # This is possibly a portion of a namespace
+            # package. Return the string representing its path,
+            # without a trailing separator.
+            return None, [f"{self.archive}{path_sep}{modpath}"]
+
+        return None, []
+
+    # Check whether we can satisfy the import of the module named by
+    # 'fullname'. Return self if we can, None if we can't.
+    def find_module(self, fullname, path=None):
+        """find_module(fullname, path=None) -> self or None.
+
+        Search for a module specified by 'fullname'. 'fullname' must be the
+        fully qualified (dotted) module name. It returns the zipimporter
+        instance itself if the module was found, or None if it wasn't.
+        The optional 'path' argument is ignored -- it's there for compatibility
+        with the importer protocol.
+        """
+        return self.find_loader(fullname, path)[0]
+
+    def get_code(self, fullname):
+        """get_code(fullname) -> code object.
+
+        Return the code object for the specified module. Raise ZipImportError
+        if the module couldn't be found.
+        """
+        code, ispackage, modpath = _get_module_code(self, fullname)
+        return code
+
+    def get_data(self, pathname):
+        """get_data(pathname) -> string with file data.
+
+        Return the data associated with 'pathname'. Raise OSError if
+        the file wasn't found.
+        """
+        if alt_path_sep:
+            pathname = pathname.replace(alt_path_sep, path_sep)
+
+        key = pathname
+        if pathname.startswith(self.archive + path_sep):
+            key = pathname[len(self.archive + path_sep) :]
+
+        try:
+            toc_entry = self._files[key]
+        except KeyError:
+            raise OSError(0, "", key)
+        return _get_data(self.archive, toc_entry)
+
+    # Return a string matching __file__ for the named module
+    def get_filename(self, fullname):
+        """get_filename(fullname) -> filename string.
+
+        Return the filename for the specified module.
+        """
+        # Deciding the filename requires working out where the code
+        # would come from if the module was actually loaded
+        code, ispackage, modpath = _get_module_code(self, fullname)
+        return modpath
+
+    def get_source(self, fullname):
+        """get_source(fullname) -> source string.
+
+        Return the source code for the specified module. Raise ZipImportError
+        if the module couldn't be found, return None if the archive does
+        contain the module, but has no source for it.
+        """
+        mi = _get_module_info(self, fullname)
+        if mi is None:
+            raise ZipImportError(f"can't find module {fullname!r}", name=fullname)
+
+        path = _get_module_path(self, fullname)
+        if mi:
+            fullpath = _bootstrap_external._path_join(path, "__init__.py")
+        else:
+            fullpath = f"{path}.py"
+
+        try:
+            toc_entry = self._files[fullpath]
+        except KeyError:
+            # we have the module, but no source
+            return None
+        return _get_data(self.archive, toc_entry).decode()
+
+    # Return a bool signifying whether the module is a package or not.
+    def is_package(self, fullname):
+        """is_package(fullname) -> bool.
+
+        Return True if the module specified by fullname is a package.
+        Raise ZipImportError if the module couldn't be found.
+        """
+        mi = _get_module_info(self, fullname)
+        if mi is None:
+            raise ZipImportError(f"can't find module {fullname!r}", name=fullname)
+        return mi
+
+    # Load and return the module named by 'fullname'.
+    def load_module(self, fullname):
+        """load_module(fullname) -> module.
+
+        Load the module specified by 'fullname'. 'fullname' must be the
+        fully qualified (dotted) module name. It returns the imported
+        module, or raises ZipImportError if it wasn't found.
+        """
+        code, ispackage, modpath = _get_module_code(self, fullname)
+        mod = sys.modules.get(fullname)
+        if mod is None or not isinstance(mod, _module_type):
+            mod = _module_type(fullname)
+            sys.modules[fullname] = mod
+        mod.__loader__ = self
+
+        try:
+            if ispackage:
+                # add __path__ to the module *before* the code gets
+                # executed
+                path = _get_module_path(self, fullname)
+                fullpath = _bootstrap_external._path_join(self.archive, path)
+                mod.__path__ = [fullpath]
+
+            if not hasattr(mod, "__builtins__"):
+                mod.__builtins__ = __builtins__
+            _bootstrap_external._fix_up_module(mod.__dict__, fullname, modpath)
+            exec(code, mod.__dict__)
+        except:
+            del sys.modules[fullname]
+            raise
+
+        try:
+            mod = sys.modules[fullname]
+        except KeyError:
+            raise ImportError(f"Loaded module {fullname!r} not found in sys.modules")
+        _bootstrap._verbose_message("import {} # loaded from Zip {}", fullname, modpath)
+        return mod
+
+    def get_resource_reader(self, fullname):
+        """Return the ResourceReader for a package in a zip file.
+
+        If 'fullname' is a package within the zip file, return the
+        'ResourceReader' object for the package.  Otherwise return None.
+        """
+        try:
+            if not self.is_package(fullname):
+                return None
+        except ZipImportError:
+            return None
+        if not _ZipImportResourceReader._registered:
+            from importlib.abc import ResourceReader
+
+            ResourceReader.register(_ZipImportResourceReader)
+            _ZipImportResourceReader._registered = True
+        return _ZipImportResourceReader(self, fullname)
+
+    def __repr__(self):
+        return f'<zipimporter object "{self.archive}{path_sep}{self.prefix}">'
+
+
+# _zip_searchorder defines how we search for a module in the Zip
+# archive: we first search for a package __init__, then for
+# non-package .pyc, and .py entries. The .pyc entries
+# are swapped by initzipimport() if we run in optimized mode. Also,
+# '/' is replaced by path_sep there.
+_zip_searchorder = (
+    (path_sep + "__init__.pyc", True, True),
+    (path_sep + "__init__.py", False, True),
+    (".pyc", True, False),
+    (".py", False, False),
+)
+
+
+# Given a module name, return the potential file path in the
+# archive (without extension).
+def _get_module_path(self, fullname):
+    return self.prefix + fullname.rpartition(".")[2]
+
+
+# Does this path represent a directory?
+def _is_dir(self, path):
+    # See if this is a "directory". If so, it's eligible to be part
+    # of a namespace package. We test by seeing if the name, with an
+    # appended path separator, exists.
+    dirpath = path + path_sep
+    # If dirpath is present in self._files, we have a directory.
+    return dirpath in self._files
+
+
+# Return some information about a module.
+def _get_module_info(self, fullname):
+    path = _get_module_path(self, fullname)
+    for suffix, isbytecode, ispackage in _zip_searchorder:
+        fullpath = path + suffix
+        if fullpath in self._files:
+            return ispackage
+    return None
+
+
+# implementation
+
+
+# _read_directory(archive) -> files dict (new reference)
+#
+# Given a path to a Zip archive, build a dict, mapping file names
+# (local to the archive, using SEP as a separator) to toc entries.
+#
+# A toc_entry is a tuple:
+#
+# (__file__,        # value to use for __file__, available for all files,
+#                   # encoded to the filesystem encoding
+#  compress,        # compression kind; 0 for uncompressed
+#  data_size,       # size of compressed data on disk
+#  file_size,       # size of decompressed data
+#  file_offset,     # offset of file header from start of archive
+#  time,            # mod time of file (in dos format)
+#  date,            # mod data of file (in dos format)
+#  crc,             # crc checksum of the data
+# )
+#
+# Directories can be recognized by the trailing path_sep in the name,
+# data_size and file_offset are 0.
+def _read_directory(archive):
+    try:
+        fp = _io.open(archive, "rb")
+    except OSError:
+        raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive)
+
+    with fp:
+        try:
+            fp.seek(-END_CENTRAL_DIR_SIZE, 2)
+            header_position = fp.tell()
+            buffer = fp.read(END_CENTRAL_DIR_SIZE)
+        except OSError:
+            raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+        if len(buffer) != END_CENTRAL_DIR_SIZE:
+            raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+        if buffer[:4] != STRING_END_ARCHIVE:
+            # Bad: End of Central Dir signature
+            # Check if there's a comment.
+            try:
+                fp.seek(0, 2)
+                file_size = fp.tell()
+            except OSError:
+                raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+            max_comment_start = max(file_size - MAX_COMMENT_LEN - END_CENTRAL_DIR_SIZE, 0)
+            try:
+                fp.seek(max_comment_start)
+                data = fp.read()
+            except OSError:
+                raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+            pos = data.rfind(STRING_END_ARCHIVE)
+            if pos < 0:
+                raise ZipImportError(f"not a Zip file: {archive!r}", path=archive)
+            buffer = data[pos : pos + END_CENTRAL_DIR_SIZE]
+            if len(buffer) != END_CENTRAL_DIR_SIZE:
+                raise ZipImportError(f"corrupt Zip file: {archive!r}", path=archive)
+            header_position = file_size - len(data) + pos
+
+        header_size = _unpack_uint32(buffer[12:16])
+        header_offset = _unpack_uint32(buffer[16:20])
+        if header_position < header_size:
+            raise ZipImportError(f"bad central directory size: {archive!r}", path=archive)
+        if header_position < header_offset:
+            raise ZipImportError(f"bad central directory offset: {archive!r}", path=archive)
+        header_position -= header_size
+        arc_offset = header_position - header_offset
+        if arc_offset < 0:
+            raise ZipImportError(
+                f"bad central directory size or offset: {archive!r}", path=archive
+            )
+
+        files = {}
+        # Start of Central Directory
+        count = 0
+        try:
+            fp.seek(header_position)
+        except OSError:
+            raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+        while True:
+            buffer = fp.read(46)
+            if len(buffer) < 4:
+                raise EOFError("EOF read where not expected")
+            # Start of file header
+            if buffer[:4] != b"PK\x01\x02":
+                break  # Bad: Central Dir File Header
+            if len(buffer) != 46:
+                raise EOFError("EOF read where not expected")
+            flags = _unpack_uint16(buffer[8:10])
+            compress = _unpack_uint16(buffer[10:12])
+            time = _unpack_uint16(buffer[12:14])
+            date = _unpack_uint16(buffer[14:16])
+            crc = _unpack_uint32(buffer[16:20])
+            data_size = _unpack_uint32(buffer[20:24])
+            file_size = _unpack_uint32(buffer[24:28])
+            name_size = _unpack_uint16(buffer[28:30])
+            extra_size = _unpack_uint16(buffer[30:32])
+            comment_size = _unpack_uint16(buffer[32:34])
+            file_offset = _unpack_uint32(buffer[42:46])
+            header_size = name_size + extra_size + comment_size
+            if file_offset > header_offset:
+                raise ZipImportError(f"bad local header offset: {archive!r}", path=archive)
+            file_offset += arc_offset
+
+            try:
+                name = fp.read(name_size)
+            except OSError:
+                raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+            if len(name) != name_size:
+                raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+            # On Windows, calling fseek to skip over the fields we don't use is
+            # slower than reading the data because fseek flushes stdio's
+            # internal buffers.    See issue #8745.
+            try:
+                if len(fp.read(header_size - name_size)) != header_size - name_size:
+                    raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+            except OSError:
+                raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+
+            if flags & 0x800:
+                # UTF-8 file names extension
+                name = name.decode()
+            else:
+                # Historical ZIP filename encoding
+                try:
+                    name = name.decode("ascii")
+                except UnicodeDecodeError:
+                    name = name.decode("latin1").translate(cp437_table)
+
+            name = name.replace("/", path_sep)
+            path = _bootstrap_external._path_join(archive, name)
+            t = (path, compress, data_size, file_size, file_offset, time, date, crc)
+            files[name] = t
+            count += 1
+    _bootstrap._verbose_message("zipimport: found {} names in {!r}", count, archive)
+    return files
+
+
+# During bootstrap, we may need to load the encodings
+# package from a ZIP file. But the cp437 encoding is implemented
+# in Python in the encodings package.
+#
+# Break out of this dependency by using the translation table for
+# the cp437 encoding.
+cp437_table = (
+    # ASCII part, 8 rows x 16 chars
+    "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f"
+    "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+    " !\"#$%&'()*+,-./"
+    "0123456789:;<=>?"
+    "@ABCDEFGHIJKLMNO"
+    "PQRSTUVWXYZ[\\]^_"
+    "`abcdefghijklmno"
+    "pqrstuvwxyz{|}~\x7f"
+    # non-ASCII part, 16 rows x 8 chars
+    "\xc7\xfc\xe9\xe2\xe4\xe0\xe5\xe7"
+    "\xea\xeb\xe8\xef\xee\xec\xc4\xc5"
+    "\xc9\xe6\xc6\xf4\xf6\xf2\xfb\xf9"
+    "\xff\xd6\xdc\xa2\xa3\xa5\u20a7\u0192"
+    "\xe1\xed\xf3\xfa\xf1\xd1\xaa\xba"
+    "\xbf\u2310\xac\xbd\xbc\xa1\xab\xbb"
+    "\u2591\u2592\u2593\u2502\u2524\u2561\u2562\u2556"
+    "\u2555\u2563\u2551\u2557\u255d\u255c\u255b\u2510"
+    "\u2514\u2534\u252c\u251c\u2500\u253c\u255e\u255f"
+    "\u255a\u2554\u2569\u2566\u2560\u2550\u256c\u2567"
+    "\u2568\u2564\u2565\u2559\u2558\u2552\u2553\u256b"
+    "\u256a\u2518\u250c\u2588\u2584\u258c\u2590\u2580"
+    "\u03b1\xdf\u0393\u03c0\u03a3\u03c3\xb5\u03c4"
+    "\u03a6\u0398\u03a9\u03b4\u221e\u03c6\u03b5\u2229"
+    "\u2261\xb1\u2265\u2264\u2320\u2321\xf7\u2248"
+    "\xb0\u2219\xb7\u221a\u207f\xb2\u25a0\xa0"
+)
+
+_importing_zlib = False
+
+
+# Return the zlib.decompress function object, or NULL if zlib couldn't
+# be imported. The function is cached when found, so subsequent calls
+# don't import zlib again.
+def _get_decompress_func():
+    global _importing_zlib
+    if _importing_zlib:
+        # Someone has a zlib.py[co] in their Zip file
+        # let's avoid a stack overflow.
+        _bootstrap._verbose_message("zipimport: zlib UNAVAILABLE")
+        raise ZipImportError("can't decompress data; zlib not available")
+
+    _importing_zlib = True
+    try:
+        from zlib import decompress
+    except Exception:
+        _bootstrap._verbose_message("zipimport: zlib UNAVAILABLE")
+        raise ZipImportError("can't decompress data; zlib not available")
+    finally:
+        _importing_zlib = False
+
+    _bootstrap._verbose_message("zipimport: zlib available")
+    return decompress
+
+
+# Given a path to a Zip file and a toc_entry, return the (uncompressed) data.
+def _get_data(archive, toc_entry):
+    datapath, compress, data_size, file_size, file_offset, time, date, crc = toc_entry
+    if data_size < 0:
+        raise ZipImportError("negative data size")
+
+    with _io.open(archive, "rb") as fp:
+        # Check to make sure the local file header is correct
+        try:
+            fp.seek(file_offset)
+        except OSError:
+            raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+        buffer = fp.read(30)
+        if len(buffer) != 30:
+            raise EOFError("EOF read where not expected")
+
+        if buffer[:4] != b"PK\x03\x04":
+            # Bad: Local File Header
+            raise ZipImportError(f"bad local file header: {archive!r}", path=archive)
+
+        name_size = _unpack_uint16(buffer[26:28])
+        extra_size = _unpack_uint16(buffer[28:30])
+        header_size = 30 + name_size + extra_size
+        file_offset += header_size  # Start of file data
+        try:
+            fp.seek(file_offset)
+        except OSError:
+            raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
+        raw_data = fp.read(data_size)
+        if len(raw_data) != data_size:
+            raise OSError("zipimport: can't read data")
+
+    if compress == 0:
+        # data is not compressed
+        return raw_data
+
+    # Decompress with zlib
+    try:
+        decompress = _get_decompress_func()
+    except Exception:
+        raise ZipImportError("can't decompress data; zlib not available")
+    return decompress(raw_data, -15)
+
+
+# Lenient date/time comparison function. The precision of the mtime
+# in the archive is lower than the mtime stored in a .pyc: we
+# must allow a difference of at most one second.
+def _eq_mtime(t1, t2):
+    # dostime only stores even seconds, so be lenient
+    return abs(t1 - t2) <= 1
+
+
+# Given the contents of a .py[co] file, unmarshal the data
+# and return the code object. Return None if it the magic word doesn't
+# match (we do this instead of raising an exception as we fall back
+# to .py if available and we don't want to mask other errors).
+def _unmarshal_code(pathname, data, mtime):
+    if len(data) < 16:
+        raise ZipImportError("bad pyc data")
+
+    if data[:4] != _bootstrap_external.MAGIC_NUMBER:
+        _bootstrap._verbose_message("{!r} has bad magic", pathname)
+        return None  # signal caller to try alternative
+
+    flags = _unpack_uint32(data[4:8])
+    if flags != 0:
+        # Hash-based pyc. We currently refuse to handle checked hash-based
+        # pycs. We could validate hash-based pycs against the source, but it
+        # seems likely that most people putting hash-based pycs in a zipfile
+        # will use unchecked ones.
+        if _imp.check_hash_based_pycs != "never" and (
+            flags != 0x1 or _imp.check_hash_based_pycs == "always"
+        ):
+            return None
+    elif mtime != 0 and not _eq_mtime(_unpack_uint32(data[8:12]), mtime):
+        _bootstrap._verbose_message("{!r} has bad mtime", pathname)
+        return None  # signal caller to try alternative
+
+    # XXX the pyc's size field is ignored; timestamp collisions are probably
+    # unimportant with zip files.
+    code = marshal.loads(data[16:])
+    if not isinstance(code, _code_type):
+        raise TypeError(f"compiled module {pathname!r} is not a code object")
+    return code
+
+
+_code_type = type(_unmarshal_code.__code__)
+
+
+# Replace any occurrences of '\r\n?' in the input string with '\n'.
+# This converts DOS and Mac line endings to Unix line endings.
+def _normalize_line_endings(source):
+    source = source.replace(b"\r\n", b"\n")
+    source = source.replace(b"\r", b"\n")
+    return source
+
+
+# Given a string buffer containing Python source code, compile it
+# and return a code object.
+def _compile_source(pathname, source):
+    source = _normalize_line_endings(source)
+    return compile(source, pathname, "exec", dont_inherit=True)
+
+
+# Convert the date/time values found in the Zip archive to a value
+# that's compatible with the time stamp stored in .pyc files.
+def _parse_dostime(d, t):
+    return time.mktime(
+        (
+            (d >> 9) + 1980,  # bits 9..15: year
+            (d >> 5) & 0xF,  # bits 5..8: month
+            d & 0x1F,  # bits 0..4: day
+            t >> 11,  # bits 11..15: hours
+            (t >> 5) & 0x3F,  # bits 8..10: minutes
+            (t & 0x1F) * 2,  # bits 0..7: seconds / 2
+            -1,
+            -1,
+            -1,
+        )
+    )
+
+
+# Given a path to a .pyc file in the archive, return the
+# modification time of the matching .py file, or 0 if no source
+# is available.
+def _get_mtime_of_source(self, path):
+    try:
+        # strip 'c' or 'o' from *.py[co]
+        assert path[-1:] in ("c", "o")
+        path = path[:-1]
+        toc_entry = self._files[path]
+        # fetch the time stamp of the .py file for comparison
+        # with an embedded pyc time stamp
+        time = toc_entry[5]
+        date = toc_entry[6]
+        return _parse_dostime(date, time)
+    except (KeyError, IndexError, TypeError):
+        return 0
+
+
+# Get the code object associated with the module specified by
+# 'fullname'.
+def _get_module_code(self, fullname):
+    path = _get_module_path(self, fullname)
+    for suffix, isbytecode, ispackage in _zip_searchorder:
+        fullpath = path + suffix
+        _bootstrap._verbose_message("trying {}{}{}", self.archive, path_sep, fullpath, verbosity=2)
+        try:
+            toc_entry = self._files[fullpath]
+        except KeyError:
+            pass
+        else:
+            modpath = toc_entry[0]
+            data = _get_data(self.archive, toc_entry)
+            if isbytecode:
+                mtime = _get_mtime_of_source(self, fullpath)
+                code = _unmarshal_code(modpath, data, mtime)
+            else:
+                code = _compile_source(modpath, data)
+            if code is None:
+                # bad magic number or non-matching mtime
+                # in byte code, try next
+                continue
+            modpath = toc_entry[0]
+            return code, ispackage, modpath
+    else:
+        raise ZipImportError(f"can't find module {fullname!r}", name=fullname)
+
+
+class _ZipImportResourceReader:
+    """Private class used to support ZipImport.get_resource_reader().
+
+    This class is allowed to reference all the innards and private parts of
+    the zipimporter.
+    """
+
+    _registered = False
+
+    def __init__(self, zipimporter, fullname):
+        self.zipimporter = zipimporter
+        self.fullname = fullname
+
+    def open_resource(self, resource):
+        fullname_as_path = self.fullname.replace(".", "/")
+        path = f"{fullname_as_path}/{resource}"
+        from io import BytesIO
+
+        try:
+            return BytesIO(self.zipimporter.get_data(path))
+        except OSError:
+            raise FileNotFoundError(path)
+
+    def resource_path(self, resource):
+        # All resources are in the zip file, so there is no path to the file.
+        # Raising FileNotFoundError tells the higher level API to extract the
+        # binary data and create a temporary file.
+        raise FileNotFoundError
+
+    def is_resource(self, name):
+        # Maybe we could do better, but if we can get the data, it's a
+        # resource.  Otherwise it isn't.
+        fullname_as_path = self.fullname.replace(".", "/")
+        path = f"{fullname_as_path}/{name}"
+        try:
+            self.zipimporter.get_data(path)
+        except OSError:
+            return False
+        return True
+
+    def contents(self):
+        # This is a bit convoluted, because fullname will be a module path,
+        # but _files is a list of file names relative to the top of the
+        # archive's namespace.  We want to compare file paths to find all the
+        # names of things inside the module represented by fullname.  So we
+        # turn the module path of fullname into a file path relative to the
+        # top of the archive, and then we iterate through _files looking for
+        # names inside that "directory".
+        from pathlib import Path
+
+        fullname_path = Path(self.zipimporter.get_filename(self.fullname))
+        relative_path = fullname_path.relative_to(self.zipimporter.archive)
+        # Don't forget that fullname names a package, so its path will include
+        # __init__.py, which we want to ignore.
+        assert relative_path.name == "__init__.py"
+        package_path = relative_path.parent
+        subdirs_seen = set()
+        for filename in self.zipimporter._files:
+            try:
+                relative = Path(filename).relative_to(package_path)
+            except ValueError:
+                continue
+            # If the path of the file (which is relative to the top of the zip
+            # namespace), relative to the package given when the resource
+            # reader was created, has a parent, then it's a name in a
+            # subdirectory and thus we skip it.
+            parent_name = relative.parent.name
+            if len(parent_name) == 0:
+                yield relative.name
+            elif parent_name not in subdirs_seen:
+                subdirs_seen.add(parent_name)
+                yield parent_name

maubot-src/maubot/loader/__init__.py

@@ -0,0 +1,3 @@
+from .abc import BasePluginLoader, IDConflictError, PluginClass, PluginLoader
+from .meta import DatabaseType, PluginMeta
+from .zip import MaubotZipImportError, ZippedPluginLoader

maubot-src/maubot/loader/abc.py

@@ -0,0 +1,98 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, TypeVar
+from abc import ABC, abstractmethod
+import asyncio
+
+from ..plugin_base import Plugin
+from .meta import PluginMeta
+
+if TYPE_CHECKING:
+    from ..instance import PluginInstance
+
+PluginClass = TypeVar("PluginClass", bound=Plugin)
+
+
+class IDConflictError(Exception):
+    pass
+
+
+class BasePluginLoader(ABC):
+    meta: PluginMeta
+
+    @property
+    @abstractmethod
+    def source(self) -> str:
+        pass
+
+    def sync_read_file(self, path: str) -> bytes:
+        raise NotImplementedError("This loader doesn't support synchronous operations")
+
+    @abstractmethod
+    async def read_file(self, path: str) -> bytes:
+        pass
+
+    def sync_list_files(self, directory: str) -> list[str]:
+        raise NotImplementedError("This loader doesn't support synchronous operations")
+
+    @abstractmethod
+    async def list_files(self, directory: str) -> list[str]:
+        pass
+
+
+class PluginLoader(BasePluginLoader, ABC):
+    id_cache: dict[str, PluginLoader] = {}
+
+    meta: PluginMeta
+    references: set[PluginInstance]
+
+    def __init__(self):
+        self.references = set()
+
+    @classmethod
+    def find(cls, plugin_id: str) -> PluginLoader:
+        return cls.id_cache[plugin_id]
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.meta.id,
+            "version": str(self.meta.version),
+            "instances": [instance.to_dict() for instance in self.references],
+        }
+
+    async def stop_instances(self) -> None:
+        await asyncio.gather(
+            *[instance.stop() for instance in self.references if instance.started]
+        )
+
+    async def start_instances(self) -> None:
+        await asyncio.gather(
+            *[instance.start() for instance in self.references if instance.enabled]
+        )
+
+    @abstractmethod
+    async def load(self) -> type[PluginClass]:
+        pass
+
+    @abstractmethod
+    async def reload(self) -> type[PluginClass]:
+        pass
+
+    @abstractmethod
+    async def delete(self) -> None:
+        pass

maubot-src/maubot/loader/meta.py

@@ -0,0 +1,69 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Optional
+
+from attr import dataclass
+from packaging.version import InvalidVersion, Version
+
+from mautrix.types import (
+    ExtensibleEnum,
+    SerializableAttrs,
+    SerializerError,
+    deserializer,
+    serializer,
+)
+
+from ..__meta__ import __version__
+
+
+@serializer(Version)
+def serialize_version(version: Version) -> str:
+    return str(version)
+
+
+@deserializer(Version)
+def deserialize_version(version: str) -> Version:
+    try:
+        return Version(version)
+    except InvalidVersion as e:
+        raise SerializerError("Invalid version") from e
+
+
+class DatabaseType(ExtensibleEnum):
+    SQLALCHEMY = "sqlalchemy"
+    ASYNCPG = "asyncpg"
+
+
+@dataclass
+class PluginMeta(SerializableAttrs):
+    id: str
+    version: Version
+    modules: List[str]
+    main_class: str
+
+    maubot: Version = Version(__version__)
+    database: bool = False
+    database_type: DatabaseType = DatabaseType.SQLALCHEMY
+    config: bool = False
+    webapp: bool = False
+    license: str = ""
+    extra_files: List[str] = []
+    dependencies: List[str] = []
+    soft_dependencies: List[str] = []
+
+    @property
+    def database_type_str(self) -> Optional[str]:
+        return self.database_type.value if self.database else None

maubot-src/maubot/loader/zip.py

@@ -0,0 +1,310 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from time import time
+from zipfile import BadZipFile, ZipFile
+import logging
+import os
+import sys
+
+from packaging.version import Version
+from ruamel.yaml import YAML, YAMLError
+
+from mautrix.types import SerializerError
+
+from ..__meta__ import __version__
+from ..config import Config
+from ..lib.zipimport import ZipImportError, zipimporter
+from ..plugin_base import Plugin
+from .abc import IDConflictError, PluginClass, PluginLoader
+from .meta import DatabaseType, PluginMeta
+
+current_version = Version(__version__)
+yaml = YAML()
+
+
+class MaubotZipImportError(Exception):
+    pass
+
+
+class MaubotZipMetaError(MaubotZipImportError):
+    pass
+
+
+class MaubotZipPreLoadError(MaubotZipImportError):
+    pass
+
+
+class MaubotZipLoadError(MaubotZipImportError):
+    pass
+
+
+class ZippedPluginLoader(PluginLoader):
+    path_cache: dict[str, ZippedPluginLoader] = {}
+    log: logging.Logger = logging.getLogger("maubot.loader.zip")
+    trash_path: str = "delete"
+    directories: list[str] = []
+
+    path: str | None
+    meta: PluginMeta | None
+    main_class: str | None
+    main_module: str | None
+    _loaded: type[PluginClass] | None
+    _importer: zipimporter | None
+    _file: ZipFile | None
+
+    def __init__(self, path: str) -> None:
+        super().__init__()
+        self.path = path
+        self.meta = None
+        self.main_class = None
+        self.main_module = None
+        self._loaded = None
+        self._importer = None
+        self._file = None
+        self._load_meta()
+        self._run_preload_checks(self._get_importer())
+        try:
+            existing = self.id_cache[self.meta.id]
+            raise IDConflictError(
+                f"Plugin with id {self.meta.id} already loaded from {existing.source}"
+            )
+        except KeyError:
+            pass
+        self.path_cache[self.path] = self
+        self.id_cache[self.meta.id] = self
+        self.log.debug(f"Preloaded plugin {self.meta.id} from {self.path}")
+
+    def to_dict(self) -> dict:
+        return {**super().to_dict(), "path": self.path}
+
+    @classmethod
+    def get(cls, path: str) -> ZippedPluginLoader:
+        path = os.path.abspath(path)
+        try:
+            return cls.path_cache[path]
+        except KeyError:
+            return cls(path)
+
+    @property
+    def source(self) -> str:
+        return self.path
+
+    def __repr__(self) -> str:
+        return (
+            "<ZippedPlugin "
+            f"path='{self.path}' "
+            f"meta={self.meta} "
+            f"loaded={self._loaded is not None}>"
+        )
+
+    def sync_read_file(self, path: str) -> bytes:
+        return self._file.read(path)
+
+    async def read_file(self, path: str) -> bytes:
+        return self.sync_read_file(path)
+
+    def sync_list_files(self, directory: str) -> list[str]:
+        directory = directory.rstrip("/")
+        return [
+            file.filename
+            for file in self._file.filelist
+            if os.path.dirname(file.filename) == directory
+        ]
+
+    async def list_files(self, directory: str) -> list[str]:
+        return self.sync_list_files(directory)
+
+    @staticmethod
+    def _read_meta(source) -> tuple[ZipFile, PluginMeta]:
+        try:
+            file = ZipFile(source)
+            data = file.read("maubot.yaml")
+        except FileNotFoundError as e:
+            raise MaubotZipMetaError("Maubot plugin not found") from e
+        except BadZipFile as e:
+            raise MaubotZipMetaError("File is not a maubot plugin") from e
+        except KeyError as e:
+            raise MaubotZipMetaError("File does not contain a maubot plugin definition") from e
+        try:
+            meta_dict = yaml.load(data)
+        except (YAMLError, KeyError, IndexError, ValueError) as e:
+            raise MaubotZipMetaError("Maubot plugin definition file is not valid YAML") from e
+        try:
+            meta = PluginMeta.deserialize(meta_dict)
+        except SerializerError as e:
+            raise MaubotZipMetaError("Maubot plugin definition in file is invalid") from e
+        if meta.maubot > current_version:
+            raise MaubotZipMetaError(
+                f"Plugin requires maubot {meta.maubot}, but this instance is {current_version}"
+            )
+        return file, meta
+
+    @classmethod
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
+        _, meta = cls._read_meta(source)
+        return meta.id, meta.version, meta.database_type if meta.database else None
+
+    def _load_meta(self) -> None:
+        file, meta = self._read_meta(self.path)
+        if self.meta and meta.id != self.meta.id:
+            raise MaubotZipMetaError("Maubot plugin ID changed during reload")
+        self.meta = meta
+        if "/" in meta.main_class:
+            self.main_module, self.main_class = meta.main_class.split("/")[:2]
+        else:
+            self.main_module = meta.modules[-1]
+            self.main_class = meta.main_class
+        self._file = file
+
+    def _get_importer(self, reset_cache: bool = False) -> zipimporter:
+        try:
+            if not self._importer or self._importer.archive != self.path:
+                self._importer = zipimporter(self.path)
+            if reset_cache:
+                self._importer.reset_cache()
+            return self._importer
+        except ZipImportError as e:
+            raise MaubotZipMetaError("File not found or not a maubot plugin") from e
+
+    def _run_preload_checks(self, importer: zipimporter) -> None:
+        try:
+            code = importer.get_code(self.main_module.replace(".", "/"))
+            if self.main_class not in code.co_names:
+                raise MaubotZipPreLoadError(
+                    f"Main class {self.main_class} not in {self.main_module}"
+                )
+        except ZipImportError as e:
+            raise MaubotZipPreLoadError(f"Main module {self.main_module} not found in file") from e
+        for module in self.meta.modules:
+            try:
+                importer.find_module(module)
+            except ZipImportError as e:
+                raise MaubotZipPreLoadError(f"Module {module} not found in file") from e
+
+    async def load(self, reset_cache: bool = False) -> type[PluginClass]:
+        try:
+            return self._load(reset_cache)
+        except MaubotZipImportError:
+            self.log.exception(f"Failed to load {self.meta.id} v{self.meta.version}")
+            raise
+
+    def _load(self, reset_cache: bool = False) -> type[PluginClass]:
+        if self._loaded is not None and not reset_cache:
+            return self._loaded
+        self._load_meta()
+        importer = self._get_importer(reset_cache=reset_cache)
+        self._run_preload_checks(importer)
+        if reset_cache:
+            self.log.debug(f"Re-preloaded plugin {self.meta.id} from {self.path}")
+        for module in self.meta.modules:
+            try:
+                importer.load_module(module)
+            except ZipImportError:
+                raise MaubotZipLoadError(f"Module {module} not found in file")
+            except Exception:
+                raise MaubotZipLoadError(f"Failed to load module {module}")
+        try:
+            main_mod = sys.modules[self.main_module]
+        except KeyError as e:
+            raise MaubotZipLoadError(f"Main module {self.main_module} of plugin not found") from e
+        try:
+            plugin = getattr(main_mod, self.main_class)
+        except AttributeError as e:
+            raise MaubotZipLoadError(f"Main class {self.main_class} of plugin not found") from e
+        if not issubclass(plugin, Plugin):
+            raise MaubotZipLoadError("Main class of plugin does not extend maubot.Plugin")
+        self._loaded = plugin
+        self.log.debug(f"Loaded and imported plugin {self.meta.id} from {self.path}")
+        return plugin
+
+    async def reload(self, new_path: str | None = None) -> type[PluginClass]:
+        self._unload()
+        if new_path is not None and new_path != self.path:
+            try:
+                del self.path_cache[self.path]
+            except KeyError:
+                pass
+            self.path = new_path
+            self.path_cache[self.path] = self
+        return await self.load(reset_cache=True)
+
+    def _unload(self) -> None:
+        for name, mod in list(sys.modules.items()):
+            if (getattr(mod, "__file__", "") or "").startswith(self.path):
+                del sys.modules[name]
+        self._loaded = None
+        self.log.debug(f"Unloaded plugin {self.meta.id} at {self.path}")
+
+    async def delete(self) -> None:
+        self._unload()
+        try:
+            del self.path_cache[self.path]
+        except KeyError:
+            pass
+        try:
+            del self.id_cache[self.meta.id]
+        except KeyError:
+            pass
+        if self._importer:
+            self._importer.remove_cache()
+            self._importer = None
+        self._loaded = None
+        self.trash(self.path, reason="delete")
+        self.meta = None
+        self.path = None
+
+    @classmethod
+    def trash(cls, file_path: str, new_name: str | None = None, reason: str = "error") -> None:
+        if cls.trash_path == "delete":
+            try:
+                os.remove(file_path)
+            except FileNotFoundError:
+                pass
+        else:
+            new_name = new_name or f"{int(time())}-{reason}-{os.path.basename(file_path)}"
+            try:
+                os.rename(file_path, os.path.abspath(os.path.join(cls.trash_path, new_name)))
+            except OSError as e:
+                cls.log.warning(f"Failed to rename {file_path}: {e} - trying to delete")
+                try:
+                    os.remove(file_path)
+                except FileNotFoundError:
+                    pass
+
+    @classmethod
+    def load_all(cls):
+        cls.log.debug("Preloading plugins...")
+        for directory in cls.directories:
+            for file in os.listdir(directory):
+                if not file.endswith(".mbp"):
+                    continue
+                path = os.path.abspath(os.path.join(directory, file))
+                try:
+                    cls.get(path)
+                except MaubotZipImportError:
+                    cls.log.exception(f"Failed to load plugin at {path}, trashing...")
+                    cls.trash(path)
+                except IDConflictError:
+                    cls.log.error(f"Duplicate plugin ID at {path}, trashing...")
+                    cls.trash(path)
+
+
+def init(config: Config) -> None:
+    ZippedPluginLoader.trash_path = config["plugin_directories.trash"]
+    ZippedPluginLoader.directories = config["plugin_directories.load"]
+    ZippedPluginLoader.load_all()

maubot-src/maubot/management/api/__init__.py

@@ -0,0 +1,48 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncio import AbstractEventLoop
+import importlib
+
+from aiohttp import web
+
+from ...config import Config
+from .auth import check_token
+from .base import get_config, routes, set_config
+from .middleware import auth, error
+
+
+@routes.get("/features")
+def features(request: web.Request) -> web.Response:
+    data = get_config()["api_features"]
+    err = check_token(request)
+    if err is None:
+        return web.json_response(data)
+    else:
+        return web.json_response(
+            {
+                "login": data["login"],
+            }
+        )
+
+
+def init(cfg: Config, loop: AbstractEventLoop) -> web.Application:
+    set_config(cfg)
+    for pkg, enabled in cfg["api_features"].items():
+        if enabled:
+            importlib.import_module(f"maubot.management.api.{pkg}")
+    app = web.Application(loop=loop, middlewares=[auth, error], client_max_size=100 * 1024 * 1024)
+    app.add_routes(routes)
+    return app

maubot-src/maubot/management/api/auth.py

@@ -0,0 +1,76 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from time import time
+
+from aiohttp import web
+
+from mautrix.types import UserID
+from mautrix.util.signed_token import sign_token, verify_token
+
+from .base import get_config, routes
+from .responses import resp
+
+
+def is_valid_token(token: str) -> bool:
+    data = verify_token(get_config()["server.unshared_secret"], token)
+    if not data:
+        return False
+    return get_config().is_admin(data.get("user_id", None))
+
+
+def create_token(user: UserID) -> str:
+    return sign_token(
+        get_config()["server.unshared_secret"],
+        {
+            "user_id": user,
+            "created_at": int(time()),
+        },
+    )
+
+
+def get_token(request: web.Request) -> str:
+    token = request.headers.get("Authorization", "")
+    if not token or not token.startswith("Bearer "):
+        token = request.query.get("access_token", "")
+    else:
+        token = token[len("Bearer ") :]
+    return token
+
+
+def check_token(request: web.Request) -> web.Response | None:
+    token = get_token(request)
+    if not token:
+        return resp.no_token
+    elif not is_valid_token(token):
+        return resp.invalid_token
+    return None
+
+
+@routes.post("/auth/ping")
+async def ping(request: web.Request) -> web.Response:
+    token = get_token(request)
+    if not token:
+        return resp.no_token
+
+    data = verify_token(get_config()["server.unshared_secret"], token)
+    if not data:
+        return resp.invalid_token
+    user = data.get("user_id", None)
+    if not get_config().is_admin(user):
+        return resp.invalid_token
+    return resp.pong(user, get_config()["api_features"])

maubot-src/maubot/management/api/base.py

@@ -0,0 +1,40 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import asyncio
+
+from aiohttp import web
+
+from ...__meta__ import __version__
+from ...config import Config
+
+routes: web.RouteTableDef = web.RouteTableDef()
+_config: Config | None = None
+
+
+def set_config(config: Config) -> None:
+    global _config
+    _config = config
+
+
+def get_config() -> Config:
+    return _config
+
+
+@routes.get("/version")
+async def version(_: web.Request) -> web.Response:
+    return web.json_response({"version": __version__})

maubot-src/maubot/management/api/client.py

@@ -0,0 +1,217 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from json import JSONDecodeError
+import logging
+
+from aiohttp import web
+
+from mautrix.client import Client as MatrixClient
+from mautrix.errors import MatrixConnectionError, MatrixInvalidToken, MatrixRequestError
+from mautrix.types import FilterID, SyncToken, UserID
+
+from ...client import Client
+from .base import routes
+from .responses import resp
+
+log = logging.getLogger("maubot.server.client")
+
+
+@routes.get("/clients")
+async def get_clients(_: web.Request) -> web.Response:
+    return resp.found([client.to_dict() for client in Client.cache.values()])
+
+
+@routes.get("/client/{id}")
+async def get_client(request: web.Request) -> web.Response:
+    user_id = request.match_info.get("id", None)
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+    return resp.found(client.to_dict())
+
+
+async def _create_client(user_id: UserID | None, data: dict) -> web.Response:
+    homeserver = data.get("homeserver", None)
+    access_token = data.get("access_token", None)
+    device_id = data.get("device_id", None)
+    new_client = MatrixClient(
+        mxid="@not:a.mxid",
+        base_url=homeserver,
+        token=access_token,
+        client_session=Client.http_client,
+    )
+    try:
+        whoami = await new_client.whoami()
+    except MatrixInvalidToken as e:
+        return resp.bad_client_access_token
+    except MatrixRequestError:
+        log.warning(f"Failed to get whoami from {homeserver} for new client", exc_info=True)
+        return resp.bad_client_access_details
+    except MatrixConnectionError:
+        log.warning(f"Failed to connect to {homeserver} for new client", exc_info=True)
+        return resp.bad_client_connection_details
+    if user_id is None:
+        existing_client = await Client.get(whoami.user_id)
+        if existing_client is not None:
+            return resp.user_exists
+    elif whoami.user_id != user_id:
+        return resp.mxid_mismatch(whoami.user_id)
+    elif whoami.device_id and device_id and whoami.device_id != device_id:
+        return resp.device_id_mismatch(whoami.device_id)
+    client = await Client.get(
+        whoami.user_id, homeserver=homeserver, access_token=access_token, device_id=device_id
+    )
+    client.enabled = data.get("enabled", True)
+    client.sync = data.get("sync", True)
+    await client.update_autojoin(data.get("autojoin", True), save=False)
+    await client.update_online(data.get("online", True), save=False)
+    client.displayname = data.get("displayname", "disable")
+    client.avatar_url = data.get("avatar_url", "disable")
+    await client.update()
+    await client.start()
+    return resp.created(client.to_dict())
+
+
+async def _update_client(client: Client, data: dict, is_login: bool = False) -> web.Response:
+    try:
+        await client.update_access_details(
+            data.get("access_token"), data.get("homeserver"), data.get("device_id")
+        )
+    except MatrixInvalidToken:
+        return resp.bad_client_access_token
+    except MatrixRequestError:
+        log.warning(
+            f"Failed to get whoami from homeserver to update client details", exc_info=True
+        )
+        return resp.bad_client_access_details
+    except MatrixConnectionError:
+        log.warning(f"Failed to connect to homeserver to update client details", exc_info=True)
+        return resp.bad_client_connection_details
+    except ValueError as e:
+        str_err = str(e)
+        if str_err.startswith("MXID mismatch"):
+            return resp.mxid_mismatch(str(e)[len("MXID mismatch: ") :])
+        elif str_err.startswith("Device ID mismatch"):
+            return resp.device_id_mismatch(str(e)[len("Device ID mismatch: ") :])
+    await client.update_avatar_url(data.get("avatar_url"), save=False)
+    await client.update_displayname(data.get("displayname"), save=False)
+    await client.update_started(data.get("started"))
+    await client.update_enabled(data.get("enabled"), save=False)
+    await client.update_autojoin(data.get("autojoin"), save=False)
+    await client.update_online(data.get("online"), save=False)
+    await client.update_sync(data.get("sync"), save=False)
+    await client.update()
+    return resp.updated(client.to_dict(), is_login=is_login)
+
+
+async def _create_or_update_client(
+    user_id: UserID, data: dict, is_login: bool = False
+) -> web.Response:
+    client = await Client.get(user_id)
+    if not client:
+        return await _create_client(user_id, data)
+    else:
+        return await _update_client(client, data, is_login=is_login)
+
+
+@routes.post("/client/new")
+async def create_client(request: web.Request) -> web.Response:
+    try:
+        data = await request.json()
+    except JSONDecodeError:
+        return resp.body_not_json
+    return await _create_client(None, data)
+
+
+@routes.put("/client/{id}")
+async def update_client(request: web.Request) -> web.Response:
+    user_id = request.match_info["id"]
+    try:
+        data = await request.json()
+    except JSONDecodeError:
+        return resp.body_not_json
+    return await _create_or_update_client(user_id, data)
+
+
+@routes.delete("/client/{id}")
+async def delete_client(request: web.Request) -> web.Response:
+    user_id = request.match_info["id"]
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+    if len(client.references) > 0:
+        return resp.client_in_use
+    if client.started:
+        await client.stop()
+    await client.delete()
+    return resp.deleted
+
+
+@routes.post("/client/{id}/clearcache")
+async def clear_client_cache(request: web.Request) -> web.Response:
+    user_id = request.match_info["id"]
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+    await client.clear_cache()
+    return resp.ok
+
+
+@routes.post("/client/{id}/verify")
+async def verify_client(request: web.Request) -> web.Response:
+    user_id = request.match_info["id"]
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+    try:
+        req = await request.json()
+    except Exception:
+        return resp.body_not_json
+    try:
+        await client.crypto.verify_with_recovery_key(req["recovery_key"])
+        client.trust_state = await client.crypto.resolve_trust(
+            client.crypto.own_identity,
+            allow_fetch=False,
+        )
+        log.debug(f"Trust state after verifying {client.id}: {client.trust_state}")
+    except Exception as e:
+        log.exception("Failed to verify client with recovery key")
+        return resp.internal_crypto_error(str(e))
+    return resp.ok
+
+
+@routes.post("/client/{id}/generate_recovery_key")
+async def generate_recovery_key(request: web.Request) -> web.Response:
+    user_id = request.match_info["id"]
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+    try:
+        keys = await client.crypto.get_own_cross_signing_public_keys()
+        if keys:
+            return resp.client_has_keys
+        key = await client.crypto.generate_recovery_key()
+        client.trust_state = await client.crypto.resolve_trust(
+            client.crypto.own_identity,
+            allow_fetch=False,
+        )
+        log.debug(f"Trust state generating recovery key for {client.id}: {client.trust_state}")
+    except Exception as e:
+        log.exception("Failed to generate recovery key for client")
+        return resp.internal_crypto_error(str(e))
+    return resp.created({"success": True, "recovery_key": key})

maubot-src/maubot/management/api/client_auth.py

@@ -0,0 +1,273 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import NamedTuple
+from http import HTTPStatus
+from json import JSONDecodeError
+import asyncio
+import hashlib
+import hmac
+import random
+import string
+
+from aiohttp import web
+from yarl import URL
+
+from mautrix.api import Method, Path, SynapseAdminPath
+from mautrix.client import ClientAPI
+from mautrix.errors import MatrixRequestError
+from mautrix.types import LoginResponse, LoginType
+
+from .base import get_config, routes
+from .client import _create_client, _create_or_update_client
+from .responses import resp
+
+
+def known_homeservers() -> dict[str, dict[str, str]]:
+    return get_config()["homeservers"]
+
+
+@routes.get("/client/auth/servers")
+async def get_known_servers(_: web.Request) -> web.Response:
+    return web.json_response({key: value["url"] for key, value in known_homeservers().items()})
+
+
+class AuthRequestInfo(NamedTuple):
+    server_name: str
+    client: ClientAPI
+    secret: str
+    username: str
+    password: str
+    user_type: str
+    device_name: str
+    update_client: bool
+    sso: bool
+
+
+truthy_strings = ("1", "true", "yes")
+
+
+async def read_client_auth_request(
+    request: web.Request,
+) -> tuple[AuthRequestInfo | None, web.Response | None]:
+    server_name = request.match_info.get("server", None)
+    server = known_homeservers().get(server_name, None)
+    if not server:
+        return None, resp.server_not_found
+    try:
+        body = await request.json()
+    except JSONDecodeError:
+        return None, resp.body_not_json
+    sso = request.query.get("sso", "").lower() in truthy_strings
+    try:
+        username = body["username"]
+        password = body["password"]
+    except KeyError:
+        if not sso:
+            return None, resp.username_or_password_missing
+        username = password = None
+    try:
+        base_url = server["url"]
+    except KeyError:
+        return None, resp.invalid_server
+    return (
+        AuthRequestInfo(
+            server_name=server_name,
+            client=ClientAPI(base_url=base_url),
+            secret=server.get("secret"),
+            username=username,
+            password=password,
+            user_type=body.get("user_type", "bot"),
+            device_name=body.get("device_name", "Maubot"),
+            update_client=request.query.get("update_client", "").lower() in truthy_strings,
+            sso=sso,
+        ),
+        None,
+    )
+
+
+def generate_mac(
+    secret: str,
+    nonce: str,
+    username: str,
+    password: str,
+    admin: bool = False,
+    user_type: str = None,
+) -> str:
+    mac = hmac.new(key=secret.encode("utf-8"), digestmod=hashlib.sha1)
+    mac.update(nonce.encode("utf-8"))
+    mac.update(b"\x00")
+    mac.update(username.encode("utf-8"))
+    mac.update(b"\x00")
+    mac.update(password.encode("utf-8"))
+    mac.update(b"\x00")
+    mac.update(b"admin" if admin else b"notadmin")
+    if user_type is not None:
+        mac.update(b"\x00")
+        mac.update(user_type.encode("utf8"))
+    return mac.hexdigest()
+
+
+@routes.post("/client/auth/{server}/register")
+async def register(request: web.Request) -> web.Response:
+    req, err = await read_client_auth_request(request)
+    if err is not None:
+        return err
+    if req.sso:
+        return resp.registration_no_sso
+    elif not req.secret:
+        return resp.registration_secret_not_found
+    path = SynapseAdminPath.v1.register
+    res = await req.client.api.request(Method.GET, path)
+    content = {
+        "nonce": res["nonce"],
+        "username": req.username,
+        "password": req.password,
+        "admin": False,
+        "user_type": req.user_type,
+    }
+    content["mac"] = generate_mac(**content, secret=req.secret)
+    try:
+        raw_res = await req.client.api.request(Method.POST, path, content=content)
+    except MatrixRequestError as e:
+        return web.json_response(
+            {
+                "errcode": e.errcode,
+                "error": e.message,
+                "http_status": e.http_status,
+            },
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+    login_res = LoginResponse.deserialize(raw_res)
+    if req.update_client:
+        return await _create_client(
+            login_res.user_id,
+            {
+                "homeserver": str(req.client.api.base_url),
+                "access_token": login_res.access_token,
+                "device_id": login_res.device_id,
+            },
+        )
+    return web.json_response(login_res.serialize())
+
+
+@routes.post("/client/auth/{server}/login")
+async def login(request: web.Request) -> web.Response:
+    req, err = await read_client_auth_request(request)
+    if err is not None:
+        return err
+    if req.sso:
+        return await _do_sso(req)
+    else:
+        return await _do_login(req)
+
+
+async def _do_sso(req: AuthRequestInfo) -> web.Response:
+    flows = await req.client.get_login_flows()
+    if not flows.supports_type(LoginType.SSO):
+        return resp.sso_not_supported
+    waiter_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16))
+    cfg = get_config()
+    public_url = (
+        URL(cfg["server.public_url"])
+        / "_matrix/maubot/v1/client/auth_external_sso/complete"
+        / waiter_id
+    )
+    sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query(
+        {"redirectUrl": str(public_url)}
+    )
+    sso_waiters[waiter_id] = req, asyncio.get_running_loop().create_future()
+    return web.json_response({"sso_url": str(sso_url), "id": waiter_id})
+
+
+async def _do_login(req: AuthRequestInfo, login_token: str | None = None) -> web.Response:
+    device_id = "".join(random.choices(string.ascii_uppercase + string.digits, k=8))
+    device_id = f"maubot_{device_id}"
+    try:
+        if req.sso:
+            res = await req.client.login(
+                token=login_token,
+                login_type=LoginType.TOKEN,
+                device_id=device_id,
+                store_access_token=False,
+                initial_device_display_name=req.device_name,
+            )
+        else:
+            res = await req.client.login(
+                identifier=req.username,
+                login_type=LoginType.PASSWORD,
+                password=req.password,
+                device_id=device_id,
+                initial_device_display_name=req.device_name,
+                store_access_token=False,
+            )
+    except MatrixRequestError as e:
+        return web.json_response(
+            {
+                "errcode": e.errcode,
+                "error": e.message,
+            },
+            status=e.http_status,
+        )
+    if req.update_client:
+        return await _create_or_update_client(
+            res.user_id,
+            {
+                "homeserver": str(req.client.api.base_url),
+                "access_token": res.access_token,
+                "device_id": res.device_id,
+            },
+            is_login=True,
+        )
+    return web.json_response(res.serialize())
+
+
+sso_waiters: dict[str, tuple[AuthRequestInfo, asyncio.Future]] = {}
+
+
+@routes.post("/client/auth/{server}/sso/{id}/wait")
+async def wait_sso(request: web.Request) -> web.Response:
+    waiter_id = request.match_info["id"]
+    req, fut = sso_waiters[waiter_id]
+    try:
+        login_token = await fut
+    finally:
+        sso_waiters.pop(waiter_id, None)
+    return await _do_login(req, login_token)
+
+
+@routes.get("/client/auth_external_sso/complete/{id}")
+async def complete_sso(request: web.Request) -> web.Response:
+    try:
+        _, fut = sso_waiters[request.match_info["id"]]
+    except KeyError:
+        return web.Response(status=404, text="Invalid session ID\n")
+    if fut.cancelled():
+        return web.Response(status=200, text="The login was cancelled from the Maubot client\n")
+    elif fut.done():
+        return web.Response(status=200, text="The login token was already received\n")
+    try:
+        fut.set_result(request.query["loginToken"])
+    except KeyError:
+        return web.Response(status=400, text="Missing loginToken query parameter\n")
+    except asyncio.InvalidStateError:
+        return web.Response(status=500, text="Invalid state\n")
+    return web.Response(
+        status=200,
+        text="Login token received, please return to your Maubot client. "
+        "This tab can be closed.\n",
+    )

maubot-src/maubot/management/api/client_proxy.py

@@ -0,0 +1,56 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import client as http, web
+
+from ...client import Client
+from .base import routes
+from .responses import resp
+
+PROXY_CHUNK_SIZE = 32 * 1024
+
+
+@routes.view("/proxy/{id}/{path:_matrix/.+}")
+async def proxy(request: web.Request) -> web.StreamResponse:
+    user_id = request.match_info.get("id", None)
+    client = await Client.get(user_id)
+    if not client:
+        return resp.client_not_found
+
+    path = request.match_info.get("path", None)
+    query = request.query.copy()
+    try:
+        del query["access_token"]
+    except KeyError:
+        pass
+    headers = request.headers.copy()
+    del headers["Host"]
+    headers["Authorization"] = f"Bearer {client.access_token}"
+    if "X-Forwarded-For" not in headers:
+        peer = request.transport.get_extra_info("peername")
+        if peer is not None:
+            host, port = peer
+            headers["X-Forwarded-For"] = f"{host}:{port}"
+
+    data = await request.read()
+    async with http.request(
+        request.method, f"{client.homeserver}/{path}", headers=headers, params=query, data=data
+    ) as proxy_resp:
+        response = web.StreamResponse(status=proxy_resp.status, headers=proxy_resp.headers)
+        await response.prepare(request)
+        async for chunk in proxy_resp.content.iter_chunked(PROXY_CHUNK_SIZE):
+            await response.write(chunk)
+        await response.write_eof()
+        return response

maubot-src/maubot/management/api/dev_open.py

@@ -0,0 +1,57 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from string import Template
+import asyncio
+import re
+
+from aiohttp import web
+from ruamel.yaml import YAML
+
+from .base import routes
+
+enabled = False
+
+
+@routes.get("/debug/open")
+async def check_enabled(_: web.Request) -> web.Response:
+    return web.json_response({"enabled": enabled})
+
+
+try:
+    yaml = YAML()
+
+    with open(".dev-open-cfg.yaml", "r") as file:
+        cfg = yaml.load(file)
+    editor_command = Template(cfg["editor"])
+    pathmap = [(re.compile(item["find"]), item["replace"]) for item in cfg["pathmap"]]
+
+    @routes.post("/debug/open")
+    async def open_file(request: web.Request) -> web.Response:
+        data = await request.json()
+        try:
+            path = data["path"]
+            for find, replace in pathmap:
+                path = find.sub(replace, path)
+            cmd = editor_command.substitute(path=path, line=data["line"])
+        except (KeyError, ValueError):
+            return web.Response(status=400)
+        res = await asyncio.create_subprocess_shell(cmd)
+        stdout, stderr = await res.communicate()
+        return web.json_response({"return": res.returncode, "stdout": stdout, "stderr": stderr})
+
+    enabled = True
+except Exception:
+    pass

maubot-src/maubot/management/api/instance.py

@@ -0,0 +1,97 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from json import JSONDecodeError
+
+from aiohttp import web
+
+from ...client import Client
+from ...instance import PluginInstance
+from ...loader import PluginLoader
+from .base import routes
+from .responses import resp
+
+
+@routes.get("/instances")
+async def get_instances(_: web.Request) -> web.Response:
+    return resp.found([instance.to_dict() for instance in PluginInstance.cache.values()])
+
+
+@routes.get("/instance/{id}")
+async def get_instance(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    if not instance:
+        return resp.instance_not_found
+    return resp.found(instance.to_dict())
+
+
+async def _create_instance(instance_id: str, data: dict) -> web.Response:
+    plugin_type = data.get("type")
+    primary_user = data.get("primary_user")
+    if not plugin_type:
+        return resp.plugin_type_required
+    elif not primary_user:
+        return resp.primary_user_required
+    elif not await Client.get(primary_user):
+        return resp.primary_user_not_found
+    try:
+        PluginLoader.find(plugin_type)
+    except KeyError:
+        return resp.plugin_type_not_found
+    instance = await PluginInstance.get(instance_id, type=plugin_type, primary_user=primary_user)
+    instance.enabled = data.get("enabled", True)
+    instance.config_str = data.get("config") or ""
+    await instance.update()
+    await instance.load()
+    await instance.start()
+    return resp.created(instance.to_dict())
+
+
+async def _update_instance(instance: PluginInstance, data: dict) -> web.Response:
+    if not await instance.update_primary_user(data.get("primary_user")):
+        return resp.primary_user_not_found
+    await instance.update_id(data.get("id"))
+    await instance.update_enabled(data.get("enabled"))
+    await instance.update_config(data.get("config"))
+    await instance.update_started(data.get("started"))
+    await instance.update_type(data.get("type"))
+    return resp.updated(instance.to_dict())
+
+
+@routes.put("/instance/{id}")
+async def update_instance(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    try:
+        data = await request.json()
+    except JSONDecodeError:
+        return resp.body_not_json
+    if not instance:
+        return await _create_instance(instance_id, data)
+    else:
+        return await _update_instance(instance, data)
+
+
+@routes.delete("/instance/{id}")
+async def delete_instance(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    if not instance:
+        return resp.instance_not_found
+    if instance.started:
+        await instance.stop()
+    await instance.delete()
+    return resp.deleted

maubot-src/maubot/management/api/instance_database.py

@@ -0,0 +1,161 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from datetime import datetime
+
+from aiohttp import web
+from asyncpg import PostgresError
+import aiosqlite
+
+from mautrix.util.async_db import Database
+
+from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
+from .base import routes
+from .responses import resp
+
+
+@routes.get("/instance/{id}/database")
+async def get_database(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    if not instance:
+        return resp.instance_not_found
+    elif not instance.inst_db:
+        return resp.plugin_has_no_database
+    return web.json_response(await instance.get_db_tables())
+
+
+@routes.get("/instance/{id}/database/{table}")
+async def get_table(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    if not instance:
+        return resp.instance_not_found
+    elif not instance.inst_db:
+        return resp.plugin_has_no_database
+    tables = await instance.get_db_tables()
+    try:
+        table = tables[request.match_info.get("table", "")]
+    except KeyError:
+        return resp.table_not_found
+    try:
+        order = [tuple(order.split(":")) for order in request.query.getall("order")]
+        order = [
+            (
+                (asc if sort.lower() == "asc" else desc)(table.columns[column])
+                if sort
+                else table.columns[column]
+            )
+            for column, sort in order
+        ]
+    except KeyError:
+        order = []
+    limit = int(request.query.get("limit", "100"))
+    if isinstance(instance.inst_db, Engine):
+        return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
+
+
+@routes.post("/instance/{id}/database/query")
+async def query(request: web.Request) -> web.Response:
+    instance_id = request.match_info["id"].lower()
+    instance = await PluginInstance.get(instance_id)
+    if not instance:
+        return resp.instance_not_found
+    elif not instance.inst_db:
+        return resp.plugin_has_no_database
+    data = await request.json()
+    try:
+        sql_query = data["query"]
+    except KeyError:
+        return resp.query_missing
+    rows_as_dict = data.get("rows_as_dict", False)
+    if isinstance(instance.inst_db, Engine):
+        return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
+    elif isinstance(instance.inst_db, Database):
+        try:
+            return await _execute_query_asyncpg(instance, sql_query, rows_as_dict)
+        except (PostgresError, aiosqlite.Error) as e:
+            return resp.sql_error(e, sql_query)
+    else:
+        return resp.unsupported_plugin_database
+
+
+def check_type(val):
+    if isinstance(val, datetime):
+        return val.isoformat()
+    return val
+
+
+async def _execute_query_asyncpg(
+    instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
+) -> web.Response:
+    data = {"ok": True, "query": sql_query}
+    if sql_query.upper().startswith("SELECT"):
+        res = await instance.inst_db.fetch(sql_query)
+        data["rows"] = [
+            (
+                {key: check_type(value) for key, value in row.items()}
+                if rows_as_dict
+                else [check_type(value) for value in row]
+            )
+            for row in res
+        ]
+        if len(res) > 0:
+            # TODO can we find column names when there are no rows?
+            data["columns"] = list(res[0].keys())
+    else:
+        res = await instance.inst_db.execute(sql_query)
+        if isinstance(res, str):
+            data["status_msg"] = res
+        elif isinstance(res, aiosqlite.Cursor):
+            data["rowcount"] = res.rowcount
+            # data["inserted_primary_key"] = res.lastrowid
+        else:
+            data["status_msg"] = "unknown status"
+    return web.json_response(data)
+
+
+def _execute_query_sqlalchemy(
+    instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
+) -> web.Response:
+    assert isinstance(instance.inst_db, Engine)
+    try:
+        res = instance.inst_db.execute(sql_query)
+    except IntegrityError as e:
+        return resp.sql_integrity_error(e, sql_query)
+    except OperationalError as e:
+        return resp.sql_operational_error(e, sql_query)
+    data = {
+        "ok": True,
+        "query": str(sql_query),
+    }
+    if res.returns_rows:
+        data["rows"] = [
+            (
+                {key: check_type(value) for key, value in row.items()}
+                if rows_as_dict
+                else [check_type(value) for value in row]
+            )
+            for row in res
+        ]
+        data["columns"] = res.keys()
+    else:
+        data["rowcount"] = res.rowcount
+    if res.is_insert:
+        data["inserted_primary_key"] = res.inserted_primary_key
+    return web.json_response(data)

maubot-src/maubot/management/api/log.py

@@ -0,0 +1,172 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from collections import deque
+from datetime import datetime
+import asyncio
+import logging
+
+from aiohttp import web, web_ws
+
+from mautrix.util import background_task
+
+from .auth import is_valid_token
+from .base import routes
+
+BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+}
+INCLUDE_ATTRS = {
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "name",
+    "pathname",
+}
+EXCLUDE_ATTRS = BUILTIN_ATTRS - INCLUDE_ATTRS
+MAX_LINES = 2048
+
+
+class LogCollector(logging.Handler):
+    lines: deque[dict]
+    formatter: logging.Formatter
+    listeners: list[web.WebSocketResponse]
+    loop: asyncio.AbstractEventLoop
+
+    def __init__(self, level=logging.NOTSET) -> None:
+        super().__init__(level)
+        self.lines = deque(maxlen=MAX_LINES)
+        self.formatter = logging.Formatter()
+        self.listeners = []
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self._emit(record)
+        except Exception as e:
+            print("Logging error:", e)
+
+    def _emit(self, record: logging.LogRecord) -> None:
+        # JSON conversion based on Marsel Mavletkulov's json-log-formatter (MIT license)
+        # https://github.com/marselester/json-log-formatter
+        content = {
+            name: value for name, value in record.__dict__.items() if name not in EXCLUDE_ATTRS
+        }
+        content["id"] = str(record.relativeCreated)
+        content["msg"] = record.getMessage()
+        content["time"] = datetime.fromtimestamp(record.created)
+
+        if record.exc_info:
+            content["exc_info"] = self.formatter.formatException(record.exc_info)
+
+        for name, value in content.items():
+            if isinstance(value, datetime):
+                content[name] = value.astimezone().isoformat()
+        asyncio.run_coroutine_threadsafe(self.send(content), loop=self.loop)
+        self.lines.append(content)
+
+    async def send(self, record: dict) -> None:
+        for ws in self.listeners:
+            try:
+                await ws.send_json(record)
+            except Exception as e:
+                print("Log sending error:", e)
+
+
+handler = LogCollector()
+log = logging.getLogger("maubot.server.websocket")
+sockets = []
+
+
+def init(loop: asyncio.AbstractEventLoop) -> None:
+    logging.root.addHandler(handler)
+    handler.loop = loop
+
+
+async def stop_all() -> None:
+    log.debug("Closing log listener websockets")
+    logging.root.removeHandler(handler)
+    for socket in sockets:
+        try:
+            await socket.close(code=1012)
+        except Exception:
+            pass
+
+
+@routes.get("/logs")
+async def log_websocket(request: web.Request) -> web.WebSocketResponse:
+    ws = web.WebSocketResponse()
+    await ws.prepare(request)
+    sockets.append(ws)
+    log.debug(f"Connection from {request.remote} opened")
+    authenticated = False
+
+    async def close_if_not_authenticated():
+        await asyncio.sleep(5)
+        if not authenticated:
+            await ws.close(code=4000)
+            log.debug(f"Connection from {request.remote} terminated due to no authentication")
+
+    background_task.create(close_if_not_authenticated())
+
+    try:
+        msg: web_ws.WSMessage
+        async for msg in ws:
+            if msg.type != web.WSMsgType.TEXT:
+                continue
+            if is_valid_token(msg.data):
+                await ws.send_json({"auth_success": True})
+                await ws.send_json({"history": list(handler.lines)})
+                if not authenticated:
+                    log.debug(f"Connection from {request.remote} authenticated")
+                    handler.listeners.append(ws)
+                    authenticated = True
+            elif not authenticated:
+                await ws.send_json({"auth_success": False})
+    except Exception:
+        try:
+            await ws.close()
+        except Exception:
+            pass
+    if authenticated:
+        handler.listeners.remove(ws)
+    log.debug(f"Connection from {request.remote} closed")
+    sockets.remove(ws)
+    return ws

maubot-src/maubot/management/api/login.py

@@ -0,0 +1,41 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import json
+
+from aiohttp import web
+
+from .auth import create_token
+from .base import get_config, routes
+from .responses import resp
+
+
+@routes.post("/auth/login")
+async def login(request: web.Request) -> web.Response:
+    try:
+        data = await request.json()
+    except json.JSONDecodeError:
+        return resp.body_not_json
+    secret = data.get("secret")
+    if secret and get_config()["server.unshared_secret"] == secret:
+        user = data.get("user") or "root"
+        return resp.logged_in(create_token(user))
+
+    username = data.get("username")
+    password = data.get("password")
+    if get_config().check_password(username, password):
+        return resp.logged_in(create_token(username))
+
+    return resp.bad_auth

maubot-src/maubot/management/api/middleware.py

@@ -0,0 +1,78 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Callable
+import base64
+import logging
+
+from aiohttp import web
+
+from .auth import check_token
+from .base import get_config
+from .responses import resp
+
+Handler = Callable[[web.Request], Awaitable[web.Response]]
+log = logging.getLogger("maubot.server")
+
+
+@web.middleware
+async def auth(request: web.Request, handler: Handler) -> web.Response:
+    subpath = request.path[len("/_matrix/maubot/v1") :]
+    if (
+        subpath.startswith("/auth/")
+        or subpath.startswith("/client/auth_external_sso/complete/")
+        or subpath == "/features"
+        or subpath == "/logs"
+    ):
+        return await handler(request)
+    err = check_token(request)
+    if err is not None:
+        return err
+    return await handler(request)
+
+
+@web.middleware
+async def error(request: web.Request, handler: Handler) -> web.Response:
+    try:
+        return await handler(request)
+    except web.HTTPException as ex:
+        if ex.status_code == 404:
+            return resp.path_not_found
+        elif ex.status_code == 405:
+            return resp.method_not_allowed
+        return web.json_response(
+            {
+                "httpexception": {
+                    "headers": {key: value for key, value in ex.headers.items()},
+                    "class": type(ex).__name__,
+                    "body": ex.text or base64.b64encode(ex.body),
+                },
+                "error": f"Unhandled HTTP {ex.status}: {ex.text[:128] or 'non-text response'}",
+                "errcode": f"unhandled_http_{ex.status}",
+            },
+            status=ex.status,
+        )
+    except Exception:
+        log.exception("Error in handler")
+        return resp.internal_server_error
+
+
+req_no = 0
+
+
+def get_req_no():
+    global req_no
+    req_no += 1
+    return req_no

maubot-src/maubot/management/api/plugin.py

@@ -0,0 +1,64 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import traceback
+
+from aiohttp import web
+
+from ...loader import MaubotZipImportError, PluginLoader
+from .base import routes
+from .responses import resp
+
+
+@routes.get("/plugins")
+async def get_plugins(_) -> web.Response:
+    return resp.found([plugin.to_dict() for plugin in PluginLoader.id_cache.values()])
+
+
+@routes.get("/plugin/{id}")
+async def get_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info["id"]
+    plugin = PluginLoader.id_cache.get(plugin_id)
+    if not plugin:
+        return resp.plugin_not_found
+    return resp.found(plugin.to_dict())
+
+
+@routes.delete("/plugin/{id}")
+async def delete_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info["id"]
+    plugin = PluginLoader.id_cache.get(plugin_id)
+    if not plugin:
+        return resp.plugin_not_found
+    elif len(plugin.references) > 0:
+        return resp.plugin_in_use
+    await plugin.delete()
+    return resp.deleted
+
+
+@routes.post("/plugin/{id}/reload")
+async def reload_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info["id"]
+    plugin = PluginLoader.id_cache.get(plugin_id)
+    if not plugin:
+        return resp.plugin_not_found
+
+    await plugin.stop_instances()
+    try:
+        await plugin.reload()
+    except MaubotZipImportError as e:
+        return resp.plugin_reload_error(str(e), traceback.format_exc())
+    await plugin.start_instances()
+    return resp.ok

maubot-src/maubot/management/api/plugin_upload.py

@@ -0,0 +1,145 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from io import BytesIO
+from time import time
+import logging
+import os.path
+import re
+import traceback
+
+from aiohttp import web
+from packaging.version import Version
+
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from .base import get_config, routes
+from .responses import resp
+
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
+log = logging.getLogger("maubot.server.upload")
+
+
+@routes.put("/plugin/{id}")
+async def put_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info["id"]
+    content = await request.read()
+    file = BytesIO(content)
+    try:
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
+    except MaubotZipImportError as e:
+        return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
+    if pid != plugin_id:
+        return resp.pid_mismatch
+    plugin = PluginLoader.id_cache.get(plugin_id, None)
+    if not plugin:
+        return await upload_new_plugin(content, pid, version)
+    elif isinstance(plugin, ZippedPluginLoader):
+        return await upload_replacement_plugin(plugin, content, version)
+    else:
+        return resp.unsupported_plugin_loader
+
+
+@routes.post("/plugins/upload")
+async def upload_plugin(request: web.Request) -> web.Response:
+    content = await request.read()
+    file = BytesIO(content)
+    try:
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
+    except MaubotZipImportError as e:
+        return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
+    plugin = PluginLoader.id_cache.get(pid, None)
+    if not plugin:
+        return await upload_new_plugin(content, pid, version)
+    elif not request.query.get("allow_override"):
+        return resp.plugin_exists
+    elif isinstance(plugin, ZippedPluginLoader):
+        return await upload_replacement_plugin(plugin, content, version)
+    else:
+        return resp.unsupported_plugin_loader
+
+
+async def upload_new_plugin(content: bytes, pid: str, version: Version) -> web.Response:
+    path = os.path.join(get_config()["plugin_directories.upload"], f"{pid}-v{version}.mbp")
+    with open(path, "wb") as p:
+        p.write(content)
+    try:
+        plugin = ZippedPluginLoader.get(path)
+    except MaubotZipImportError as e:
+        ZippedPluginLoader.trash(path)
+        return resp.plugin_import_error(str(e), traceback.format_exc())
+    return resp.created(plugin.to_dict())
+
+
+async def upload_replacement_plugin(
+    plugin: ZippedPluginLoader, content: bytes, new_version: Version
+) -> web.Response:
+    dirname = os.path.dirname(plugin.path)
+    old_filename = os.path.basename(plugin.path)
+    if str(plugin.meta.version) in old_filename:
+        replacement = (
+            str(new_version)
+            if plugin.meta.version != new_version
+            else f"{new_version}-ts{int(time() * 1000)}"
+        )
+        filename = re.sub(
+            f"{re.escape(str(plugin.meta.version))}(-ts[0-9]+)?", replacement, old_filename
+        )
+    else:
+        filename = old_filename.rstrip(".mbp")
+        filename = f"{filename}-v{new_version}.mbp"
+    path = os.path.join(dirname, filename)
+    with open(path, "wb") as p:
+        p.write(content)
+    old_path = plugin.path
+    await plugin.stop_instances()
+    try:
+        await plugin.reload(new_path=path)
+    except MaubotZipImportError as e:
+        log.exception(f"Error loading updated version of {plugin.meta.id}, rolling back")
+        try:
+            await plugin.reload(new_path=old_path)
+            await plugin.start_instances()
+        except MaubotZipImportError:
+            log.warning(f"Failed to roll back update of {plugin.meta.id}", exc_info=True)
+        finally:
+            ZippedPluginLoader.trash(path, reason="failed_update")
+        return resp.plugin_import_error(str(e), traceback.format_exc())
+    try:
+        await plugin.start_instances()
+    except Exception as e:
+        log.exception(f"Error starting {plugin.meta.id} instances after update, rolling back")
+        try:
+            await plugin.stop_instances()
+            await plugin.reload(new_path=old_path)
+            await plugin.start_instances()
+        except Exception:
+            log.warning(f"Failed to roll back update of {plugin.meta.id}", exc_info=True)
+        finally:
+            ZippedPluginLoader.trash(path, reason="failed_update")
+        return resp.plugin_reload_error(str(e), traceback.format_exc())
+
+    log.debug(f"Successfully updated {plugin.meta.id}, moving old version to trash")
+    ZippedPluginLoader.trash(old_path, reason="update")
+    return resp.updated(plugin.to_dict())

maubot-src/maubot/management/api/responses.py

@@ -0,0 +1,510 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from http import HTTPStatus
+
+from aiohttp import web
+from asyncpg import PostgresError
+import aiosqlite
+
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
+
+class _Response:
+    @property
+    def body_not_json(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Request body is not JSON",
+                "errcode": "body_not_json",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def plugin_type_required(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Plugin type is required when creating plugin instances",
+                "errcode": "plugin_type_required",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def primary_user_required(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Primary user is required when creating plugin instances",
+                "errcode": "primary_user_required",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def bad_client_access_token(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Invalid access token",
+                "errcode": "bad_client_access_token",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def bad_client_access_details(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Invalid homeserver or access token",
+                "errcode": "bad_client_access_details",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def bad_client_connection_details(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Could not connect to homeserver",
+                "errcode": "bad_client_connection_details",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    def mxid_mismatch(self, found: str) -> web.Response:
+        return web.json_response(
+            {
+                "error": (
+                    "The Matrix user ID of the client and the user ID of the access token don't "
+                    f"match. Access token is for user {found}"
+                ),
+                "errcode": "mxid_mismatch",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    def device_id_mismatch(self, found: str) -> web.Response:
+        return web.json_response(
+            {
+                "error": (
+                    "The Matrix device ID of the client and the device ID of the access token "
+                    f"don't match. Access token is for device {found}"
+                ),
+                "errcode": "mxid_mismatch",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def pid_mismatch(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "The ID in the path does not match the ID of the uploaded plugin",
+                "errcode": "pid_mismatch",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def username_or_password_missing(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Username or password missing",
+                "errcode": "username_or_password_missing",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def query_missing(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Query missing",
+                "errcode": "query_missing",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @staticmethod
+    def sql_error(error: PostgresError | aiosqlite.Error, query: str) -> web.Response:
+        return web.json_response(
+            {
+                "ok": False,
+                "query": query,
+                "error": str(error),
+                "errcode": "sql_error",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @staticmethod
+    def sql_operational_error(error: OperationalError, query: str) -> web.Response:
+        return web.json_response(
+            {
+                "ok": False,
+                "query": query,
+                "error": str(error.orig),
+                "full_error": str(error),
+                "errcode": "sql_operational_error",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @staticmethod
+    def sql_integrity_error(error: IntegrityError, query: str) -> web.Response:
+        return web.json_response(
+            {
+                "ok": False,
+                "query": query,
+                "error": str(error.orig),
+                "full_error": str(error),
+                "errcode": "sql_integrity_error",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def bad_auth(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Invalid username or password",
+                "errcode": "invalid_auth",
+            },
+            status=HTTPStatus.UNAUTHORIZED,
+        )
+
+    @property
+    def no_token(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Authorization token missing",
+                "errcode": "auth_token_missing",
+            },
+            status=HTTPStatus.UNAUTHORIZED,
+        )
+
+    @property
+    def invalid_token(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Invalid authorization token",
+                "errcode": "auth_token_invalid",
+            },
+            status=HTTPStatus.UNAUTHORIZED,
+        )
+
+    @property
+    def plugin_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Plugin not found",
+                "errcode": "plugin_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def client_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Client not found",
+                "errcode": "client_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def primary_user_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Client for given primary user not found",
+                "errcode": "primary_user_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def instance_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Plugin instance not found",
+                "errcode": "instance_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def plugin_type_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Given plugin type not found",
+                "errcode": "plugin_type_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def path_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Resource not found",
+                "errcode": "resource_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def server_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Registration target server not found",
+                "errcode": "server_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def registration_secret_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Config does not have a registration secret for that server",
+                "errcode": "registration_secret_not_found",
+            },
+            status=HTTPStatus.NOT_FOUND,
+        )
+
+    @property
+    def registration_no_sso(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "The register operation is only for registering with a password",
+                "errcode": "registration_no_sso",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def sso_not_supported(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "That server does not seem to support single sign-on",
+                "errcode": "sso_not_supported",
+            },
+            status=HTTPStatus.FORBIDDEN,
+        )
+
+    @property
+    def plugin_has_no_database(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Given plugin does not have a database",
+                "errcode": "plugin_has_no_database",
+            }
+        )
+
+    @property
+    def unsupported_plugin_database(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "The database type is not supported by this API",
+                "errcode": "unsupported_plugin_database",
+            }
+        )
+
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
+    @property
+    def table_not_found(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Given table not found in plugin database",
+                "errcode": "table_not_found",
+            }
+        )
+
+    @property
+    def method_not_allowed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Method not allowed",
+                "errcode": "method_not_allowed",
+            },
+            status=HTTPStatus.METHOD_NOT_ALLOWED,
+        )
+
+    @property
+    def user_exists(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "There is already a client with the user ID of that token",
+                "errcode": "user_exists",
+            },
+            status=HTTPStatus.CONFLICT,
+        )
+
+    @property
+    def plugin_exists(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "A plugin with the same ID as the uploaded plugin already exists",
+                "errcode": "plugin_exists",
+            },
+            status=HTTPStatus.CONFLICT,
+        )
+
+    @property
+    def plugin_in_use(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Plugin instances of this type still exist",
+                "errcode": "plugin_in_use",
+            },
+            status=HTTPStatus.PRECONDITION_FAILED,
+        )
+
+    @property
+    def client_in_use(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Plugin instances with this client as their primary user still exist",
+                "errcode": "client_in_use",
+            },
+            status=HTTPStatus.PRECONDITION_FAILED,
+        )
+
+    @staticmethod
+    def plugin_import_error(error: str, stacktrace: str) -> web.Response:
+        return web.json_response(
+            {
+                "error": error,
+                "stacktrace": stacktrace,
+                "errcode": "plugin_invalid",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @staticmethod
+    def plugin_reload_error(error: str, stacktrace: str) -> web.Response:
+        return web.json_response(
+            {
+                "error": error,
+                "stacktrace": stacktrace,
+                "errcode": "plugin_reload_fail",
+            },
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+    @property
+    def internal_server_error(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Internal server error",
+                "errcode": "internal_server_error",
+            },
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+    @property
+    def invalid_server(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Invalid registration server object in maubot configuration",
+                "errcode": "invalid_server",
+            },
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+    @property
+    def unsupported_plugin_loader(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Existing plugin with same ID uses unsupported plugin loader",
+                "errcode": "unsupported_plugin_loader",
+            },
+            status=HTTPStatus.BAD_REQUEST,
+        )
+
+    @property
+    def client_has_keys(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Client already has cross-signing keys",
+                "errcode": "client_has_keys",
+            },
+            status=HTTPStatus.CONFLICT,
+        )
+
+    def internal_crypto_error(self, message: str) -> web.Response:
+        return web.json_response(
+            {
+                "error": f"Internal crypto error: {message}",
+                "errcode": "internal_crypto_error",
+            },
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+    @property
+    def not_implemented(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "Not implemented",
+                "errcode": "not_implemented",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
+    @property
+    def ok(self) -> web.Response:
+        return web.json_response(
+            {"success": True},
+            status=HTTPStatus.OK,
+        )
+
+    @property
+    def deleted(self) -> web.Response:
+        return web.Response(status=HTTPStatus.NO_CONTENT)
+
+    @staticmethod
+    def found(data: dict) -> web.Response:
+        return web.json_response(data, status=HTTPStatus.OK)
+
+    @staticmethod
+    def updated(data: dict, is_login: bool = False) -> web.Response:
+        return web.json_response(data, status=HTTPStatus.ACCEPTED if is_login else HTTPStatus.OK)
+
+    def logged_in(self, token: str) -> web.Response:
+        return self.found({"token": token})
+
+    def pong(self, user: str, features: dict) -> web.Response:
+        return self.found({"username": user, "features": features})
+
+    @staticmethod
+    def created(data: dict) -> web.Response:
+        return web.json_response(data, status=HTTPStatus.CREATED)
+
+
+resp = _Response()

maubot-src/maubot/management/api/spec.md

@@ -0,0 +1,2 @@
+# Maubot Management API
+This document has been moved to docs.mau.fi: <https://docs.mau.fi/maubot/management-api.html>

maubot-src/maubot/management/api/spec.yaml

@@ -0,0 +1,728 @@
+openapi: 3.0.0
+info:
+  title: Maubot Management
+  version: 0.1.0
+  description: The API to manage a [maubot](https://github.com/maubot/maubot) instance
+  license:
+    name: GNU Affero General Public License version 3
+    url: 'https://github.com/maubot/maubot/blob/master/LICENSE'
+security:
+- bearer: []
+servers:
+- url: /_matrix/maubot/v1
+
+paths:
+  /auth/login:
+    post:
+      operationId: login
+      summary: Log in with the unshared secret or username+password
+      tags: [Authentication]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              description: Set either username+password or secret.
+              properties:
+                secret:
+                  type: string
+                  description: The unshared server secret for root login
+                username:
+                  type: string
+                  description: The username for normal login
+                password:
+                  type: string
+                  description: The password for normal login
+      responses:
+        200:
+          description: Logged in successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  token:
+                    type: string
+        401:
+          description: Invalid credentials
+  /auth/ping:
+    post:
+      operationId: ping
+      summary: Check if the given token is valid
+      tags: [Authentication]
+      responses:
+        200:
+          description: Token is OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  username:
+                    type: string
+        401:
+          description: Token is not OK
+
+  /plugins:
+    get:
+      operationId: get_plugins
+      summary: Get the list of installed plugins
+      tags: [Plugins]
+      responses:
+        200:
+          description: The list of plugins
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/Plugin'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+  /plugins/upload:
+    post:
+      operationId: upload_plugin
+      summary: Upload a new plugin
+      description: Upload a new plugin. If the plugin already exists, enabled instances will be restarted.
+      tags: [Plugins]
+      parameters:
+      - name: allow_override
+        in: query
+        description: Set to allow overriding existing plugins
+        required: false
+        schema:
+          type: boolean
+          default: false
+      requestBody:
+        content:
+          application/zip:
+            schema:
+              type: string
+              format: binary
+              example: The plugin maubot archive (.mbp)
+      responses:
+        200:
+          description: Plugin uploaded and replaced current version successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Plugin'
+        201:
+          description: New plugin uploaded successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Plugin'
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        409:
+          description: Plugin already exists and allow_override was not specified.
+  '/plugin/{id}':
+    parameters:
+    - name: id
+      in: path
+      description: The ID of the plugin to get
+      required: true
+      schema:
+        type: string
+    get:
+      operationId: get_plugin
+      summary: Get information about a specific plugin
+      tags: [Plugins]
+      responses:
+        200:
+          description: Plugin found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Plugin'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/PluginNotFound'
+    delete:
+      operationId: delete_plugin
+      summary: Delete a plugin
+      description: Delete a plugin. All instances of the plugin must be deleted before deleting the plugin.
+      tags: [Plugins]
+      responses:
+        204:
+          description: Plugin deleted
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/PluginNotFound'
+        412:
+          description: One or more plugin instances of this type exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+    put:
+      operationId: put_plugin
+      summary: Upload a new or replacement plugin
+      description: |
+        Upload a new or replacement plugin with the specified ID.
+        A HTTP 400 will be returned if the ID of the uploaded plugin
+        doesn't match the ID in the path. If the plugin already
+        exists, enabled instances will be restarted.
+      tags: [Plugins]
+      requestBody:
+        content:
+          application/zip:
+            schema:
+              type: string
+              format: binary
+              example: The plugin maubot archive (.mbp)
+      responses:
+        200:
+          description: Plugin uploaded and replaced current version successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Plugin'
+        201:
+          description: New plugin uploaded successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Plugin'
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+  /plugin/{id}/reload:
+    parameters:
+    - name: id
+      in: path
+      description: The ID of the plugin to get
+      required: true
+      schema:
+        type: string
+    post:
+      operationId: reload_plugin
+      summary: Reload a plugin from disk
+      tags: [Plugins]
+      responses:
+        200:
+          description: Plugin reloaded
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/PluginNotFound'
+
+  /instances:
+    get:
+      operationId: get_instances
+      summary: Get all plugin instances
+      tags: [Plugin instances]
+      responses:
+        200:
+          description: The list of instances
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/PluginInstance'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+  '/instance/{id}':
+    parameters:
+    - name: id
+      in: path
+      description: The ID of the instance to get
+      required: true
+      schema:
+        type: string
+    get:
+      operationId: get_instance
+      summary: Get information about a specific plugin instance
+      tags: [Plugin instances]
+      responses:
+        200:
+          description: Plugin instance found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PluginInstance'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/InstanceNotFound'
+    delete:
+      operationId: delete_instance
+      summary: Delete a specific plugin instance
+      tags: [Plugin instances]
+      responses:
+        204:
+          description: Plugin instance deleted
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/InstanceNotFound'
+    put:
+      operationId: update_instance
+      summary: Create a plugin instance or edit the details of an existing plugin instance
+      tags: [Plugin instances]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PluginInstance'
+      responses:
+        200:
+          description: Plugin instance edited
+        201:
+          description: Plugin instance created
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          description: The referenced client or plugin type could not be found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+  '/clients':
+    get:
+      operationId: get_clients
+      summary: Get the list of Matrix clients
+      tags: [Clients]
+      responses:
+        200:
+          description: The list of plugins
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/MatrixClient'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+  /client/new:
+    post:
+      operationId: create_client
+      summary: Create a Matrix client
+      tags: [Clients]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/MatrixClient'
+      responses:
+        201:
+          description: Client created
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/ClientNotFound'
+        409:
+          description: There is already a client with the user ID of that token.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+  '/client/{id}':
+    parameters:
+    - name: id
+      in: path
+      description: The Matrix user ID of the client to get
+      required: true
+      schema:
+        type: string
+    get:
+      operationId: get_client
+      summary: Get information about a specific Matrix client
+      tags: [Clients]
+      responses:
+        200:
+          description: Client found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/ClientNotFound'
+    put:
+      operationId: update_client
+      summary: Create or update a Matrix client
+      tags: [Clients]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/MatrixClient'
+      responses:
+        202:
+          description: Client updated
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        201:
+          description: Client created
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+    delete:
+      operationId: delete_client
+      summary: Delete a Matrix client
+      tags: [Clients]
+      responses:
+        204:
+          description: Client deleted
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/ClientNotFound'
+        412:
+          description: One or more plugin instances with this as their primary client exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+  '/client/{id}/clearcache':
+    parameters:
+    - name: id
+      in: path
+      description: The Matrix user ID of the client to change
+      required: true
+      schema:
+        type: string
+    put:
+      operationId: clear_client_cache
+      summary: Clear the sync/state cache of a Matrix client
+      tags: [Clients]
+      responses:
+        200:
+          description: Cache cleared
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  success:
+                    type: boolean
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/ClientNotFound'
+  /client/auth/servers:
+    get:
+      operationId: get_client_auth_servers
+      summary: Get the list of servers you can register or log in on via the maubot server
+      tags: [Clients]
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                description: Key-value map from server name to homeserver URL
+                additionalProperties:
+                  type: string
+                  description: The homeserver URL
+                example:
+                  maunium.net: https://maunium.net
+                  example.com: https://matrix.example.org
+        401:
+          $ref: '#/components/responses/Unauthorized'
+  '/client/auth/{server}/register':
+    parameters:
+    - name: server
+      in: path
+      description: The server name to register the account on.
+      required: true
+      schema:
+        type: string
+    - name: update_client
+      in: query
+      description: Should maubot store the access details in a Client instead of returning them?
+      required: false
+      schema:
+        type: boolean
+    post:
+      operationId: client_auth_register
+      summary: |
+        Register a new account on the given Matrix server using the shared registration
+        secret configured into the maubot server.
+      tags: [Clients]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/MatrixAuthentication'
+      responses:
+        200:
+          description: Registration successful
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  access_token:
+                    type: string
+                    example: syt_123_456_789
+                  user_id:
+                    type: string
+                    example: '@putkiteippi:maunium.net'
+                  device_id:
+                    type: string
+                    example: maubot_F00BAR12
+        201:
+          description: Client created (when update_client is true)
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        409:
+          description: |
+            There is already a client with the user ID of that token.
+            This should usually not happen, because the user ID was just created.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        500:
+          $ref: '#/components/responses/MatrixServerError'
+  '/client/auth/{server}/login':
+    parameters:
+    - name: server
+      in: path
+      description: The server name to log in to.
+      required: true
+      schema:
+        type: string
+    - name: update_client
+      in: query
+      description: Should maubot store the access details in a Client instead of returning them?
+      required: false
+      schema:
+        type: boolean
+    post:
+      operationId: client_auth_login
+      summary: Log in to the given Matrix server via the maubot server
+      tags: [Clients]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/MatrixAuthentication'
+      responses:
+        200:
+          description: Login successful
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  user_id:
+                    type: string
+                    example: '@putkiteippi:maunium.net'
+                  access_token:
+                    type: string
+                    example: syt_123_456_789
+                  device_id:
+                    type: string
+                    example: maubot_F00BAR12
+        201:
+          description: Client created (when update_client is true)
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        202:
+          description: Client updated (when update_client is true)
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MatrixClient'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        500:
+          $ref: '#/components/responses/MatrixServerError'
+
+components:
+  responses:
+    Unauthorized:
+      description: Invalid or missing access token
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
+    PluginNotFound:
+      description: Plugin not found
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
+    ClientNotFound:
+      description: Client not found
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
+    InstanceNotFound:
+      description: Plugin instance not found
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
+    BadRequest:
+      description: Bad request (e.g. bad request body)
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/Error'
+    MatrixServerError:
+      description: The Matrix server returned an error
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              errcode:
+                type: string
+                description: The `errcode` returned by the server.
+              error:
+                type: string
+                description: The human-readable error returned by the server.
+              http_status:
+                type: integer
+                description: The HTTP status returned by the server.
+
+  securitySchemes:
+    bearer:
+      type: http
+      scheme: bearer
+      description: Required authentication for all endpoints
+  schemas:
+    Error:
+      type: object
+      properties:
+        error:
+          type: string
+          description: A human-readable error message
+        errcode:
+          type: string
+          description: A simple error code
+    Plugin:
+      type: object
+      properties:
+        id:
+          type: string
+          example: xyz.maubot.jesaribot
+        version:
+          type: string
+          example: 2.0.0
+        instances:
+          type: array
+          items:
+            $ref: '#/components/schemas/PluginInstance'
+    PluginInstance:
+      type: object
+      properties:
+        id:
+          type: string
+          example: jesaribot
+        type:
+          type: string
+          example: xyz.maubot.jesaribot
+        enabled:
+          type: boolean
+          example: true
+        started:
+          type: boolean
+          example: true
+        primary_user:
+          type: string
+          example: '@putkiteippi:maunium.net'
+        config:
+          type: string
+          example: "YAML"
+    MatrixClient:
+      type: object
+      properties:
+        id:
+          type: string
+          example: '@putkiteippi:maunium.net'
+          readOnly: true
+          description: The Matrix user ID of this client.
+        homeserver:
+          type: string
+          example: 'https://maunium.net'
+          description: The homeserver URL for this client.
+        access_token:
+          type: string
+          description: The Matrix access token for this client.
+        device_id:
+          type: string
+          description: The Matrix device ID corresponding to the access token.
+        fingerprint:
+          type: string
+          description: The encryption device fingerprint for verification.
+        enabled:
+          type: boolean
+          example: true
+          description: Whether or not this client is enabled.
+        started:
+          type: boolean
+          example: true
+          description: Whether or not this client and its instances have been started.
+        sync:
+          type: boolean
+          example: true
+          description: Whether or not syncing is enabled on this client.
+        sync_ok:
+          type: boolean
+          example: true
+          description: Whether or not the previous sync was successful on this client.
+        autojoin:
+          type: boolean
+          example: true
+          description: Whether or not this client should automatically join rooms when invited.
+        displayname:
+          type: string
+          example: J. E. Saarinen
+          description: The display name for this client.
+        avatar_url:
+          type: string
+          example: 'mxc://maunium.net/FsPQQTntCCqhJMFtwArmJdaU'
+          description: The content URI of the avatar for this client.
+        instances:
+          type: array
+          readOnly: true
+          items:
+            $ref: '#/components/schemas/PluginInstance'
+    MatrixAuthentication:
+      type: object
+      properties:
+        username:
+          type: string
+          example: putkiteippi
+          description: The user ID localpart to register/log in as.
+        password:
+          type: string
+          example: p455w0rd
+          description: The password for/of the user.

maubot-src/maubot/management/frontend/.eslintrc.json

@@ -0,0 +1,67 @@
+{
+  "extends": "react-app",
+  "plugins": [
+    "import"
+  ],
+  "rules": {
+    "indent": ["error", 4, {
+      "ignoredNodes": [
+        "JSXAttribute",
+        "JSXSpreadAttribute"
+      ],
+      "FunctionDeclaration": {"parameters": "first"},
+      "FunctionExpression": {"parameters": "first"},
+      "CallExpression": {"arguments": "first"},
+      "ArrayExpression": "first",
+      "ObjectExpression": "first",
+      "ImportDeclaration": "first"
+    }],
+    "react/jsx-indent-props": ["error", "first"],
+    "object-curly-newline": ["error", {
+      "consistent": true
+    }],
+    "object-curly-spacing": ["error", "always", {
+      "arraysInObjects": false,
+      "objectsInObjects": false
+    }],
+    "array-bracket-spacing": ["error", "never"],
+    "one-var": ["error", {
+      "initialized": "never",
+      "uninitialized": "always"
+    }],
+    "one-var-declaration-per-line": ["error", "initializations"],
+    "quotes": ["error", "double"],
+    "semi": ["error", "never"],
+    "comma-dangle": ["error", "always-multiline"],
+    "max-len": ["warn", 100],
+    "space-before-function-paren": ["error", {
+      "anonymous": "never",
+      "named": "never",
+      "asyncArrow": "always"
+    }],
+    "func-style": ["warn", "declaration", {"allowArrowFunctions": true}],
+    "id-length": ["warn", {"max": 40, "exceptions": ["i", "j", "x", "y", "_"]}],
+    "arrow-body-style": ["error", "as-needed"],
+    "new-cap": ["warn", {
+      "newIsCap": true,
+      "capIsNew": true
+    }],
+    "no-empty": ["error", {
+      "allowEmptyCatch": true
+    }],
+    "eol-last": ["error", "always"],
+    "no-console": "off",
+    "import/no-nodejs-modules": "error",
+    "import/order": ["warn", {
+      "groups": [
+        "builtin",
+        "external",
+        "internal",
+        "parent",
+        "sibling",
+        "index"
+      ],
+      "newlines-between": "never"
+    }]
+  }
+}

maubot-src/maubot/management/frontend/.gitignore

@@ -0,0 +1,5 @@
+/node_modules
+/build
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

maubot-src/maubot/management/frontend/.sass-lint.yml

@@ -0,0 +1,27 @@
+options:
+  merge-default-rules: false
+  formatter: html
+  max-warnings: 50
+
+files:
+  include: 'src/style/**/*.sass'
+
+rules:
+  extends-before-mixins: 2
+  extends-before-declarations: 2
+  placeholder-in-extend: 2
+  mixins-before-declarations:
+    - 2
+    - exclude:
+        - breakpoint
+        - mq
+  no-warn: 1
+  no-debug: 1
+  hex-notation:
+    - 2
+    - style: uppercase
+  indentation:
+    - 2
+    - size: 4
+  property-sort-order:
+    - 0

maubot-src/maubot/management/frontend/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "maubot-manager",
+  "version": "0.1.1",
+  "private": true,
+  "author": "Tulir Asokan",
+  "license": "AGPL-3.0-or-later",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/maubot/maubot.git"
+  },
+  "bugs": {
+    "url": "https://github.com/maubot/maubot/issues"
+  },
+  "homepage": ".",
+  "dependencies": {
+    "react": "^17.0.2",
+    "react-ace": "^9.4.1",
+    "react-contextmenu": "^2.14.0",
+    "react-dom": "^17.0.2",
+    "react-json-tree": "^0.16.1",
+    "react-router-dom": "^5.3.0",
+    "react-scripts": "5.0.0",
+    "react-select": "^5.2.1",
+    "sass": "^1.34.1"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject"
+  },
+  "browserslist": [
+    "last 2 firefox versions",
+    "last 2 and_ff versions",
+    "last 2 chrome versions",
+    "last 2 and_chr versions",
+    "last 1 safari versions",
+    "last 1 ios_saf versions"
+  ]
+}

maubot-src/maubot/management/frontend/public/index.html

@@ -0,0 +1,37 @@
+<!--
+maubot - A plugin-based Matrix bot system.
+Copyright (C) 2022 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.png">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <meta name="theme-color" content="#50D367">
+    <meta property="og:title" content="Maubot Manager"/>
+    <meta property="og:description" content="Maubot management interface"/>
+    <meta property="og:image" content="%PUBLIC_URL%/favicon.png"/>
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json">
+    <title>Maubot Manager</title>
+</head>
+<body>
+<noscript>
+    You need to enable JavaScript to run this app.
+</noscript>
+<div id="root"></div>
+</body>
+</html>

maubot-src/maubot/management/frontend/public/manifest.json

@@ -0,0 +1,15 @@
+{
+  "short_name": "Maubot",
+  "name": "Maubot Manager",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 48x48 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#50D367",
+  "background_color": "#FAFAFA"
+}

maubot-src/maubot/management/frontend/src/api.js

@@ -0,0 +1,276 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+let BASE_PATH = "/_matrix/maubot/v1"
+
+export function setBasePath(basePath) {
+    BASE_PATH = basePath
+}
+
+function getHeaders(contentType = "application/json") {
+    return {
+        "Content-Type": contentType,
+        "Authorization": `Bearer ${localStorage.accessToken}`,
+    }
+}
+
+async function defaultDelete(type, id) {
+    const resp = await fetch(`${BASE_PATH}/${type}/${id}`, {
+        headers: getHeaders(),
+        method: "DELETE",
+    })
+    if (resp.status === 204) {
+        return {
+            "success": true,
+        }
+    }
+    return await resp.json()
+}
+
+async function defaultPut(type, entry, id = undefined, suffix = undefined) {
+    const resp = await fetch(`${BASE_PATH}/${type}/${id || entry.id}${suffix || ""}`, {
+        headers: getHeaders(),
+        body: JSON.stringify(entry),
+        method: "PUT",
+    })
+    return await resp.json()
+}
+
+async function defaultGet(path) {
+    const resp = await fetch(`${BASE_PATH}${path}`, { headers: getHeaders() })
+    return await resp.json()
+}
+
+export async function login(username, password) {
+    const resp = await fetch(`${BASE_PATH}/auth/login`, {
+        method: "POST",
+        body: JSON.stringify({
+            username,
+            password,
+        }),
+    })
+    return await resp.json()
+}
+
+let features = null
+
+export async function ping() {
+    const response = await fetch(`${BASE_PATH}/auth/ping`, {
+        method: "POST",
+        headers: getHeaders(),
+    })
+    const json = await response.json()
+    if (json.username) {
+        features = json.features
+        return json.username
+    } else if (json.errcode === "auth_token_missing" || json.errcode === "auth_token_invalid") {
+        if (!features) {
+            await remoteGetFeatures()
+        }
+        return null
+    }
+    throw json
+}
+
+export const remoteGetFeatures = async () => {
+    features = await defaultGet("/features")
+}
+
+export const getFeatures = () => features
+
+export async function openLogSocket() {
+    let protocol = window.location.protocol === "https:" ? "wss:" : "ws:"
+    const url = `${protocol}//${window.location.host}${BASE_PATH}/logs`
+    const wrapper = {
+        socket: null,
+        connected: false,
+        authenticated: false,
+        onLog: data => undefined,
+        onHistory: history => undefined,
+        fails: -1,
+    }
+    const openHandler = () => {
+        wrapper.socket.send(localStorage.accessToken)
+        wrapper.connected = true
+    }
+    const messageHandler = evt => {
+        // TODO use logs
+        const data = JSON.parse(evt.data)
+        if (data.auth_success !== undefined) {
+            if (data.auth_success) {
+                console.info("Websocket connection authentication successful")
+                wrapper.authenticated = true
+                wrapper.fails = -1
+            } else {
+                console.info("Websocket connection authentication failed")
+            }
+        } else if (data.history) {
+            wrapper.onHistory(data.history)
+        } else {
+            wrapper.onLog(data)
+        }
+    }
+    const closeHandler = evt => {
+        if (evt) {
+            if (evt.code === 4000) {
+                console.error("Websocket connection failed: access token invalid or not provided")
+            } else if (evt.code === 1012) {
+                console.info("Websocket connection closed: server is restarting")
+            }
+        }
+        wrapper.connected = false
+        wrapper.socket = null
+        wrapper.fails++
+        const SECOND = 1000
+        setTimeout(() => {
+            wrapper.socket = new WebSocket(url)
+            wrapper.socket.onopen = openHandler
+            wrapper.socket.onmessage = messageHandler
+            wrapper.socket.onclose = closeHandler
+        }, Math.min(wrapper.fails * 5 * SECOND, 30 * SECOND))
+    }
+
+    closeHandler()
+
+    return wrapper
+}
+
+let _debugOpenFileEnabled = undefined
+export const debugOpenFileEnabled = () => _debugOpenFileEnabled
+export const updateDebugOpenFileEnabled = async () => {
+    const resp = await defaultGet("/debug/open")
+    _debugOpenFileEnabled = resp["enabled"] || false
+}
+
+export async function debugOpenFile(path, line) {
+    const resp = await fetch(`${BASE_PATH}/debug/open`, {
+        headers: getHeaders(),
+        body: JSON.stringify({ path, line }),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+export const getInstances = () => defaultGet("/instances")
+export const getInstance = id => defaultGet(`/instance/${id}`)
+export const putInstance = (instance, id) => defaultPut("instance", instance, id)
+export const deleteInstance = id => defaultDelete("instance", id)
+
+export const getInstanceDatabase = id => defaultGet(`/instance/${id}/database`)
+export const queryInstanceDatabase = async (id, query) => {
+    const resp = await fetch(`${BASE_PATH}/instance/${id}/database/query`, {
+        headers: getHeaders(),
+        body: JSON.stringify({ query }),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+export const getPlugins = () => defaultGet("/plugins")
+export const getPlugin = id => defaultGet(`/plugin/${id}`)
+export const deletePlugin = id => defaultDelete("plugin", id)
+
+export async function uploadPlugin(data, id) {
+    let resp
+    if (id) {
+        resp = await fetch(`${BASE_PATH}/plugin/${id}`, {
+            headers: getHeaders("application/zip"),
+            body: data,
+            method: "PUT",
+        })
+    } else {
+        resp = await fetch(`${BASE_PATH}/plugins/upload`, {
+            headers: getHeaders("application/zip"),
+            body: data,
+            method: "POST",
+        })
+    }
+    return await resp.json()
+}
+
+export const getClients = () => defaultGet("/clients")
+export const getClient = id => defaultGet(`/clients/${id}`)
+
+export async function uploadAvatar(id, data, mime) {
+    const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, {
+        headers: getHeaders(mime),
+        body: data,
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+export function getAvatarURL({ id, avatar_url }) {
+    if (!avatar_url?.startsWith("mxc://")) {
+        return null
+    }
+    avatar_url = avatar_url.substring("mxc://".length)
+    // Note: the maubot backend will replace the query param with an authorization header
+    return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${
+        localStorage.accessToken}`
+}
+
+export const putClient = client => defaultPut("client", client)
+export const deleteClient = id => defaultDelete("client", id)
+
+export async function clearClientCache(id) {
+    const resp = await fetch(`${BASE_PATH}/client/${id}/clearcache`, {
+        headers: getHeaders(),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+export async function verifyClient(id, recovery_key) {
+    const resp = await fetch(`${BASE_PATH}/client/${id}/verify`, {
+        headers: getHeaders(),
+        method: "POST",
+        body: JSON.stringify({ recovery_key }),
+    })
+    return await resp.json()
+}
+
+export async function generateRecoveryKey(id) {
+    const resp = await fetch(`${BASE_PATH}/client/${id}/generate_recovery_key`, {
+        headers: getHeaders(),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+export const getClientAuthServers = () => defaultGet("/client/auth/servers")
+
+export async function doClientAuth(server, type, username, password) {
+    const resp = await fetch(`${BASE_PATH}/client/auth/${server}/${type}`, {
+        headers: getHeaders(),
+        body: JSON.stringify({ username, password }),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
+// eslint-disable-next-line import/no-anonymous-default-export
+export default {
+    login, ping, setBasePath, getFeatures, remoteGetFeatures,
+    openLogSocket,
+    debugOpenFile, debugOpenFileEnabled, updateDebugOpenFileEnabled,
+    getInstances, getInstance, putInstance, deleteInstance,
+    getInstanceDatabase, queryInstanceDatabase,
+    getPlugins, getPlugin, uploadPlugin, deletePlugin,
+    getClients, getClient, uploadAvatar, getAvatarURL, putClient, deleteClient, clearClientCache,
+    verifyClient, generateRecoveryKey,
+    getClientAuthServers, doClientAuth,
+}

maubot-src/maubot/management/frontend/src/components/PreferenceTable.js

@@ -0,0 +1,71 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import Select from "react-select"
+import CreatableSelect from "react-select/creatable"
+import Switch from "./Switch"
+
+export const PrefTable = ({ children, wrapperClass }) => {
+    if (wrapperClass) {
+        return (
+            <div className={wrapperClass}>
+                <div className="preference-table">
+                    {children}
+                </div>
+            </div>
+        )
+    }
+    return (
+        <div className="preference-table">
+            {children}
+        </div>
+    )
+}
+
+export const PrefRow =
+    ({ name, fullWidth = false, labelFor = undefined, changed = false, children }) => (
+        <div className={`entry ${fullWidth ? "full-width" : ""} ${changed ? "changed" : ""}`}>
+            <label htmlFor={labelFor}>{name}</label>
+            <div className="value">{children}</div>
+        </div>
+    )
+
+export const PrefInput = ({ rowName, value, origValue, fullWidth = false, ...args }) => (
+    <PrefRow name={rowName} fullWidth={fullWidth} labelFor={rowName}
+             changed={origValue !== undefined && value !== origValue}>
+        <input {...args} value={value} id={rowName}/>
+    </PrefRow>
+)
+
+export const PrefSwitch = ({ rowName, active, origActive, fullWidth = false, ...args }) => (
+    <PrefRow name={rowName} fullWidth={fullWidth} labelFor={rowName}
+             changed={origActive !== undefined && active !== origActive}>
+        <Switch {...args} active={active} id={rowName}/>
+    </PrefRow>
+)
+
+export const PrefSelect = ({
+    rowName, value, origValue, fullWidth = false, creatable = false, ...args
+}) => (
+    <PrefRow name={rowName} fullWidth={fullWidth} labelFor={rowName}
+             changed={origValue !== undefined && value.id !== origValue}>
+        {creatable
+            ? <CreatableSelect className="select" {...args} id={rowName} value={value}/>
+            : <Select className="select" {...args} id={rowName} value={value}/>}
+    </PrefRow>
+)
+
+export default PrefTable

maubot-src/maubot/management/frontend/src/components/PrivateRoute.js

@@ -0,0 +1,28 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import { Route, Redirect } from "react-router-dom"
+
+const PrivateRoute = ({ component, render, authed, to = "/login", ...args }) => (
+    <Route
+        {...args}
+        render={(props) => authed === true
+            ? (component ? React.createElement(component, props) : render())
+            : <Redirect to={{ pathname: to }}/>}
+    />
+)
+
+export default PrivateRoute

maubot-src/maubot/management/frontend/src/components/Spinner.js

@@ -0,0 +1,11 @@
+import React from "react"
+
+const Spinner = (props) => (
+    <div {...props} className={`spinner ${props["className"] || ""}`}>
+        <svg viewBox="25 25 50 50">
+            <circle cx="50" cy="50" r="20" fill="none" strokeWidth="2" strokeMiterlimit="10"/>
+        </svg>
+    </div>
+)
+
+export default Spinner

maubot-src/maubot/management/frontend/src/components/Switch.js

@@ -0,0 +1,59 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+
+class Switch extends Component {
+    constructor(props) {
+        super(props)
+        this.state = {
+            active: props.active,
+        }
+    }
+
+    componentDidUpdate(prevProps) {
+        if (prevProps.active !== this.props.active) {
+            this.setState({
+                active: this.props.active,
+            })
+        }
+    }
+
+    toggle = () => {
+        if (this.props.onToggle) {
+            this.props.onToggle(!this.state.active)
+        } else {
+            this.setState({ active: !this.state.active })
+        }
+    }
+
+    toggleKeyboard = evt => (evt.key === " " || evt.key === "Enter") && this.toggle()
+
+    render() {
+        return (
+            <div className="switch" data-active={this.state.active} onClick={this.toggle}
+                 tabIndex="0" onKeyPress={this.toggleKeyboard} id={this.props.id}>
+                <div className="box">
+                    <span className="text">
+                        <span className="on">{this.props.onText || "On"}</span>
+                        <span className="off">{this.props.offText || "Off"}</span>
+                    </span>
+                </div>
+            </div>
+        )
+    }
+}
+
+export default Switch

maubot-src/maubot/management/frontend/src/index.js

@@ -0,0 +1,21 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import ReactDOM from "react-dom"
+import "./style/index.sass"
+import App from "./pages/Main"
+
+ReactDOM.render(<App/>, document.getElementById("root"))

maubot-src/maubot/management/frontend/src/pages/Login.js

@@ -0,0 +1,72 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+import Spinner from "../components/Spinner"
+import api from "../api"
+
+class Login extends Component {
+    constructor(props, context) {
+        super(props, context)
+        this.state = {
+            username: "",
+            password: "",
+            loading: false,
+            error: "",
+        }
+    }
+
+    inputChanged = event => this.setState({ [event.target.name]: event.target.value })
+
+    login = async evt => {
+        evt.preventDefault()
+        this.setState({ loading: true })
+        const resp = await api.login(this.state.username, this.state.password)
+        if (resp.token) {
+            await this.props.onLogin(resp.token)
+        } else if (resp.error) {
+            this.setState({ error: resp.error, loading: false })
+        } else {
+            this.setState({ error: "Unknown error", loading: false })
+            console.log("Unknown error:", resp)
+        }
+    }
+
+    render() {
+        if (!api.getFeatures().login) {
+            return <div className="login-wrapper">
+                <div className="login errored">
+                    <h1>Maubot Manager</h1>
+                    <div className="error">Login has been disabled in the maubot config.</div>
+                </div>
+            </div>
+        }
+        return <div className="login-wrapper">
+            <form className={`login ${this.state.error && "errored"}`} onSubmit={this.login}>
+                <h1>Maubot Manager</h1>
+                <input type="text" placeholder="Username" value={this.state.username}
+                       name="username" onChange={this.inputChanged}/>
+                <input type="password" placeholder="Password" value={this.state.password}
+                       name="password" onChange={this.inputChanged}/>
+                <button onClick={this.login} type="submit">
+                    {this.state.loading ? <Spinner/> : "Log in"}
+                </button>
+                {this.state.error && <div className="error">{this.state.error}</div>}
+            </form>
+        </div>
+    }
+}
+
+export default Login

maubot-src/maubot/management/frontend/src/pages/Main.js

@@ -0,0 +1,91 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+import { HashRouter as Router, Switch } from "react-router-dom"
+import PrivateRoute from "../components/PrivateRoute"
+import Spinner from "../components/Spinner"
+import api from "../api"
+import Dashboard from "./dashboard"
+import Login from "./Login"
+
+class Main extends Component {
+    constructor(props) {
+        super(props)
+        this.state = {
+            pinged: false,
+            authed: false,
+        }
+    }
+
+    async componentDidMount() {
+        await this.getBasePath()
+        if (localStorage.accessToken) {
+            await this.ping()
+        } else {
+            await api.remoteGetFeatures()
+        }
+        this.setState({ pinged: true })
+    }
+
+    async getBasePath() {
+        try {
+            const resp = await fetch(process.env.PUBLIC_URL + "/paths.json", {
+                headers: { "Content-Type": "application/json" },
+            })
+            const apiPaths = await resp.json()
+            api.setBasePath(apiPaths.api_path)
+        } catch (err) {
+            console.error("Failed to get API path:", err)
+        }
+    }
+
+
+    async ping() {
+        try {
+            const username = await api.ping()
+            if (username) {
+                localStorage.username = username
+                this.setState({ authed: true })
+            } else {
+                delete localStorage.accessToken
+            }
+        } catch (err) {
+            console.error(err)
+        }
+    }
+
+    login = async (token) => {
+        localStorage.accessToken = token
+        await this.ping()
+    }
+
+    render() {
+        if (!this.state.pinged) {
+            return <Spinner className="maubot-loading"/>
+        }
+        return <Router>
+            <div className={`maubot-wrapper ${this.state.authed ? "authenticated" : ""}`}>
+                <Switch>
+                    <PrivateRoute path="/login" render={() => <Login onLogin={this.login}/>}
+                                  authed={!this.state.authed} to="/"/>
+                    <PrivateRoute path="/" component={Dashboard} authed={this.state.authed}/>
+                </Switch>
+            </div>
+        </Router>
+    }
+}
+
+export default Main

maubot-src/maubot/management/frontend/src/pages/dashboard/BaseMainView.js

@@ -0,0 +1,98 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+import { Link } from "react-router-dom"
+import api from "../../api"
+
+class BaseMainView extends Component {
+    constructor(props) {
+        super(props)
+        this.state = Object.assign(this.initialState, props.entry)
+    }
+
+    UNSAFE_componentWillReceiveProps(nextProps) {
+        const newState = Object.assign(this.initialState, nextProps.entry)
+        for (const key of this.entryKeys) {
+            if (this.props.entry[key] === nextProps.entry[key]) {
+                newState[key] = this.state[key]
+            }
+        }
+        this.setState(newState)
+    }
+
+    delete = async () => {
+        if (!window.confirm(`Are you sure you want to delete ${this.state.id}?`)) {
+            return
+        }
+        this.setState({ deleting: true })
+        const resp = await this.deleteFunc(this.state.id)
+        if (resp.success) {
+            this.props.history.push("/")
+            this.props.onDelete()
+        } else {
+            this.setState({ deleting: false, error: resp.error })
+        }
+    }
+
+    get entryKeys() {
+        return []
+    }
+
+    get initialState() {
+        return {}
+    }
+
+    get hasInstances() {
+        return this.state.instances && this.state.instances.length > 0
+    }
+
+    get isNew() {
+        return !this.props.entry.id
+    }
+
+    inputChange = event => {
+        if (!event.target.name) {
+            return
+        }
+        this.setState({ [event.target.name]: event.target.value })
+    }
+
+    async readFile(file) {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader()
+            reader.readAsArrayBuffer(file)
+            reader.onload = evt => resolve(evt.target.result)
+            reader.onerror = err => reject(err)
+        })
+    }
+
+    renderInstances = () => !this.isNew && (
+        <div className="instances">
+            <h3>{this.hasInstances ? "Instances" : "No instances :("}</h3>
+            {this.state.instances.map(instance => (
+                <Link className="instance" key={instance.id} to={`/instance/${instance.id}`}>
+                    {instance.id}
+                </Link>
+            ))}
+        </div>
+    )
+
+    renderLogButton = (filter) => !this.isNew && api.getFeatures().log && <div className="buttons">
+        <button className="open-log" onClick={() => this.props.openLog(filter)}>View logs</button>
+    </div>
+}
+
+export default BaseMainView

maubot-src/maubot/management/frontend/src/pages/dashboard/Client.js

@@ -0,0 +1,392 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import { NavLink, withRouter } from "react-router-dom"
+import { ReactComponent as ChevronRight } from "../../res/chevron-right.svg"
+import { ReactComponent as UploadButton } from "../../res/upload.svg"
+import { ReactComponent as NoAvatarIcon } from "../../res/bot.svg"
+import { PrefTable, PrefSwitch, PrefInput, PrefSelect } from "../../components/PreferenceTable"
+import Spinner from "../../components/Spinner"
+import api from "../../api"
+import BaseMainView from "./BaseMainView"
+
+const ClientListEntry = ({ entry }) => {
+    const classes = ["client", "entry"]
+    if (!entry.enabled) {
+        classes.push("disabled")
+    } else if (!entry.started) {
+        classes.push("stopped")
+    }
+    const avatarMXC = entry.avatar_url === "disable"
+        ? entry.remote_avatar_url
+        : entry.avatar_url
+    const avatarURL = avatarMXC && api.getAvatarURL({
+        id: entry.id,
+        avatar_url: avatarMXC,
+    })
+    const displayname = (
+        entry.displayname === "disable"
+            ? entry.remote_displayname
+            : entry.displayname
+    ) || entry.id
+    return (
+        <NavLink className={classes.join(" ")} to={`/client/${entry.id}`}>
+            {avatarURL
+                ? <img className='avatar' src={avatarURL} alt=""/>
+                : <NoAvatarIcon className='avatar'/>}
+            <span className="displayname">{displayname}</span>
+            <ChevronRight className='chevron'/>
+        </NavLink>
+    )
+}
+
+class Client extends BaseMainView {
+    static ListEntry = ClientListEntry
+
+    constructor(props) {
+        super(props)
+        this.deleteFunc = api.deleteClient
+        this.recovery_key = null
+        this.homeserverOptions = []
+    }
+
+    get entryKeys() {
+        return ["id", "displayname", "homeserver", "avatar_url", "access_token", "device_id",
+                "sync", "autojoin", "online", "enabled", "started", "trust_state"]
+    }
+
+    get initialState() {
+        return {
+            id: "",
+            displayname: "disable",
+            homeserver: "",
+            avatar_url: "disable",
+            access_token: "",
+            device_id: "",
+            fingerprint: null,
+            sync: true,
+            autojoin: true,
+            enabled: true,
+            online: true,
+            started: false,
+            trust_state: null,
+
+            instances: [],
+
+            uploadingAvatar: false,
+            saving: false,
+            deleting: false,
+            verifying: false,
+            startingOrStopping: false,
+            clearingCache: false,
+            error: "",
+        }
+    }
+
+    get clientInState() {
+        const client = Object.assign({}, this.state)
+        delete client.uploadingAvatar
+        delete client.saving
+        delete client.deleting
+        delete client.startingOrStopping
+        delete client.clearingCache
+        delete client.error
+        delete client.instances
+        return client
+    }
+
+    get selectedHomeserver() {
+        return this.state.homeserver
+            ? this.homeserverEntry([this.props.ctx.homeserversByURL[this.state.homeserver],
+                                    this.state.homeserver])
+            : {}
+    }
+
+    homeserverEntry = ([serverName, serverURL]) => serverURL && {
+        id: serverURL,
+        value: serverURL,
+        label: serverName || serverURL,
+    }
+
+    componentDidUpdate(prevProps) {
+        this.updateHomeserverOptions()
+    }
+
+    updateHomeserverOptions() {
+        this.homeserverOptions = Object
+            .entries(this.props.ctx.homeserversByName)
+            .map(this.homeserverEntry)
+    }
+
+    isValidHomeserver(value) {
+        try {
+            return Boolean(new URL(value))
+        } catch (err) {
+            return false
+        }
+    }
+
+    avatarUpload = async event => {
+        const file = event.target.files[0]
+        this.setState({
+            uploadingAvatar: true,
+        })
+        const data = await this.readFile(file)
+        const resp = await api.uploadAvatar(this.state.id, data, file.type)
+        this.setState({
+            uploadingAvatar: false,
+            avatar_url: resp.content_uri,
+        })
+    }
+
+    save = async () => {
+        this.setState({ saving: true })
+        const resp = await api.putClient(this.clientInState)
+        if (resp.id) {
+            if (this.isNew) {
+                this.props.history.push(`/client/${resp.id}`)
+            } else {
+                this.setState({ saving: false, error: "" })
+            }
+            this.props.onChange(resp)
+        } else {
+            this.setState({ saving: false, error: resp.error })
+        }
+    }
+
+    startOrStop = async () => {
+        this.setState({ startingOrStopping: true })
+        const resp = await api.putClient({
+            id: this.props.entry.id,
+            started: !this.props.entry.started,
+        })
+        if (resp.id) {
+            this.props.onChange(resp)
+            this.setState({ startingOrStopping: false, error: "" })
+        } else {
+            this.setState({ startingOrStopping: false, error: resp.error })
+        }
+    }
+
+    clearCache = async () => {
+        this.setState({ clearingCache: true })
+        const resp = await api.clearClientCache(this.props.entry.id)
+        if (resp.success) {
+            this.setState({ clearingCache: false, error: "" })
+        } else {
+            this.setState({ clearingCache: false, error: resp.error })
+        }
+    }
+
+    verifyRecoveryKey = async () => {
+        const recoveryKey = window.prompt("Enter recovery key")
+        if (!recoveryKey) {
+            return
+        }
+        this.setState({ verifying: true })
+        const resp = await api.verifyClient(this.props.entry.id, recoveryKey)
+        if (resp.success) {
+            this.setState({ verifying: false, error: "" })
+        } else {
+            this.setState({ verifying: false, error: resp.error })
+        }
+    }
+
+    generateRecoveryKey = async () => {
+        this.setState({ verifying: true })
+        const resp = await api.generateRecoveryKey(this.props.entry.id)
+        if (resp.success) {
+            this.recovery_key = resp.recovery_key
+            this.setState({ verifying: false, error: "" })
+        } else {
+            this.setState({ verifying: false, error: resp.error })
+        }
+    }
+
+    get loading() {
+        return this.state.saving || this.state.startingOrStopping
+            || this.clearingCache || this.state.deleting || this.state.verifying
+    }
+
+    renderStartedContainer = () => {
+        let text
+        if (this.props.entry.started) {
+            if (this.props.entry.sync_ok) {
+                text = "Started"
+            } else {
+                text = "Erroring"
+            }
+        } else if (this.props.entry.enabled) {
+            text = "Stopped"
+        } else {
+            text = "Disabled"
+        }
+        return <div className="started-container">
+            <span className={`started ${this.props.entry.started}
+                             ${this.props.entry.sync_ok ? "sync_ok" : "sync_error"}
+                             ${this.props.entry.enabled ? "" : "disabled"}`}/>
+            <span className="text">{text}</span>
+        </div>
+    }
+
+    get avatarMXC() {
+        if (this.state.avatar_url === "disable") {
+            return this.props.entry.remote_avatar_url
+        } else if (!this.state.avatar_url?.startsWith("mxc://")) {
+            return null
+        }
+        return this.state.avatar_url
+    }
+
+    get avatarURL() {
+        return api.getAvatarURL({
+            id: this.state.id,
+            avatar_url: this.avatarMXC,
+        })
+    }
+
+    renderSidebar = () => !this.isNew && (
+        <div className="sidebar">
+            <div className={`avatar-container ${this.avatarMXC ? "" : "no-avatar"}
+                        ${this.state.uploadingAvatar ? "uploading" : ""}`}>
+                {this.avatarMXC && <img className="avatar" src={this.avatarURL} alt="Avatar"/>}
+                <UploadButton className="upload"/>
+                <input className="file-selector" type="file" accept="image/png, image/jpeg"
+                       onChange={this.avatarUpload} disabled={this.state.uploadingAvatar}
+                       onDragEnter={evt => evt.target.parentElement.classList.add("drag")}
+                       onDragLeave={evt => evt.target.parentElement.classList.remove("drag")}/>
+                {this.state.uploadingAvatar && <Spinner/>}
+            </div>
+            {this.renderStartedContainer()}
+            {(this.props.entry.started || this.props.entry.enabled) && <>
+                <button className="save" onClick={this.startOrStop} disabled={this.loading}>
+                    {this.state.startingOrStopping ? <Spinner/>
+                        : (this.props.entry.started ? "Stop" : "Start")}
+                </button>
+                <button className="clearcache" onClick={this.clearCache} disabled={this.loading}>
+                    {this.state.clearingCache ? <Spinner/> : "Clear cache"}
+                </button>
+            </>}
+        </div>
+    )
+
+    renderPreferences = () => (
+        <PrefTable>
+            <PrefInput rowName="User ID" type="text" disabled={!this.isNew} fullWidth={true}
+                       name={this.isNew ? "id" : ""} className="id"
+                       value={this.state.id} origValue={this.props.entry.id}
+                       placeholder="@fancybot:example.com" onChange={this.inputChange}/>
+            {api.getFeatures().client_auth ? (
+                <PrefSelect rowName="Homeserver" options={this.homeserverOptions} fullWidth={true}
+                            isSearchable={true} value={this.selectedHomeserver}
+                            origValue={this.props.entry.homeserver}
+                            onChange={({ value }) => this.setState({ homeserver: value })}
+                            creatable={true} isValidNewOption={this.isValidHomeserver}/>
+            ) : (
+                <PrefInput rowName="Homeserver" type="text" name="homeserver" fullWidth={true}
+                           value={this.state.homeserver} origValue={this.props.entry.homeserver}
+                           placeholder="https://example.com" onChange={this.inputChange}/>
+            )}
+            <PrefInput rowName="Access token" type="text" name="access_token"
+                       value={this.state.access_token} origValue={this.props.entry.access_token}
+                       placeholder="syt_bWF1Ym90_tUleVHiGyLKwXLaAMqlm_0afdcq"
+                       onChange={this.inputChange}/>
+            <PrefInput rowName="Device ID" type="text" name="device_id"
+                       value={this.state.device_id || ""}
+                       origValue={this.props.entry.device_id || ""}
+                       placeholder="maubot_F00BAR12" onChange={this.inputChange}/>
+            {this.props.entry.fingerprint && <>
+                <PrefInput
+                    rowName="E2EE device fingerprint" type="text" disabled={true}
+                    value={this.props.entry.fingerprint} className="fingerprint"
+                />
+                <PrefInput
+                    rowName="Trust state" type="text" disabled={true}
+                    value={this.props.entry.trust_state || ""} className="trust-state"
+                />
+            </>}
+            <PrefInput rowName="Display name" type="text" name="displayname"
+                       value={this.state.displayname} origValue={this.props.entry.displayname}
+                       placeholder="My fancy bot" onChange={this.inputChange}/>
+            <PrefInput rowName="Avatar URL" type="text" name="avatar_url"
+                       value={this.state.avatar_url} origValue={this.props.entry.avatar_url}
+                       placeholder="mxc://example.com/mbmwyoTvPhEQPiCskcUsppko"
+                       onChange={this.inputChange}/>
+            <PrefSwitch rowName="Sync"
+                        active={this.state.sync} origActive={this.props.entry.sync}
+                        onToggle={sync => this.setState({ sync })}/>
+            <PrefSwitch rowName="Autojoin"
+                        active={this.state.autojoin} origActive={this.props.entry.autojoin}
+                        onToggle={autojoin => this.setState({ autojoin })}/>
+            <PrefSwitch rowName="Enabled"
+                        active={this.state.enabled} origActive={this.props.entry.enabled}
+                        onToggle={enabled => this.setState({
+                            enabled,
+                            started: enabled && this.state.started,
+                        })}/>
+            <PrefSwitch rowName="Online"
+                        active={this.state.online} origActive={this.props.entry.online}
+                        onToggle={online => this.setState({ online })} />
+        </PrefTable>
+    )
+
+    renderPrefButtons = () => <>
+        <div className="buttons">
+            {!this.isNew && (
+                <button className={`delete ${this.hasInstances ? "disabled-bg" : ""}`}
+                        onClick={this.delete} disabled={this.loading || this.hasInstances}
+                        title={this.hasInstances ? "Can't delete client that is in use" : ""}>
+                    {this.state.deleting ? <Spinner/> : "Delete"}
+                </button>
+            )}
+            <button className="save" onClick={this.save} disabled={this.loading}>
+                {this.state.saving ? <Spinner/> : (this.isNew ? "Create" : "Save")}
+            </button>
+        </div>
+        {this.renderLogButton(this.state.id)}
+        <div className="error">{this.state.error}</div>
+    </>
+
+    renderVerifyButtons = () => <>
+        <div className="buttons">
+            <button className="verify" onClick={this.verifyRecoveryKey} disabled={this.loading}>
+                {this.state.verifying ? <Spinner/> : "Verify"}
+            </button>
+            <button className="verify" onClick={this.generateRecoveryKey} disabled={this.loading}>
+                {this.state.verifying ? <Spinner/> : "Generate recovery key"}
+            </button>
+        </div>
+        {this.recovery_key && <div className="recovery-key">
+            <strong>Recovery key:</strong> <code>{this.recovery_key}</code>
+        </div>}
+    </>
+
+    render() {
+        return <>
+            <div className="client">
+                {this.renderSidebar()}
+                <div className="info">
+                    {this.renderPreferences()}
+                    {this.renderPrefButtons()}
+                    {this.props.entry.fingerprint && this.renderVerifyButtons()}
+                    {this.renderInstances()}
+                </div>
+            </div>
+        </>
+    }
+}
+
+export default withRouter(Client)

maubot-src/maubot/management/frontend/src/pages/dashboard/Home.js

@@ -0,0 +1,30 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+
+class Home extends Component {
+    render() {
+        return <>
+            <div className="home">
+                See sidebar to get started
+            </div>
+            <div className="buttons">
+            </div>
+        </>
+    }
+}
+
+export default Home

maubot-src/maubot/management/frontend/src/pages/dashboard/Instance.js

@@ -0,0 +1,233 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import { Link, NavLink, Route, Switch, withRouter } from "react-router-dom"
+import AceEditor from "react-ace"
+import "ace-builds/src-noconflict/mode-yaml"
+import "ace-builds/src-noconflict/theme-github"
+import { ReactComponent as ChevronRight } from "../../res/chevron-right.svg"
+import { ReactComponent as NoAvatarIcon } from "../../res/bot.svg"
+import PrefTable, { PrefInput, PrefSelect, PrefSwitch } from "../../components/PreferenceTable"
+import api from "../../api"
+import Spinner from "../../components/Spinner"
+import BaseMainView from "./BaseMainView"
+import InstanceDatabase from "./InstanceDatabase"
+
+const InstanceListEntry = ({ entry }) => (
+    <NavLink className="instance entry" to={`/instance/${entry.id}`}>
+        <span className="id">{entry.id}</span>
+        <ChevronRight className='chevron'/>
+    </NavLink>
+)
+
+class Instance extends BaseMainView {
+    static ListEntry = InstanceListEntry
+
+    constructor(props) {
+        super(props)
+        this.deleteFunc = api.deleteInstance
+        this.updateClientOptions()
+    }
+
+    get entryKeys() {
+        return ["id", "primary_user", "enabled", "started", "type", "config", "database_engine"]
+    }
+
+    get initialState() {
+        return {
+            id: "",
+            primary_user: "",
+            enabled: true,
+            started: true,
+            type: "",
+            config: "",
+            database_engine: "",
+
+            saving: false,
+            deleting: false,
+            error: "",
+        }
+    }
+
+    get instanceInState() {
+        const instance = Object.assign({}, this.state)
+        delete instance.saving
+        delete instance.deleting
+        delete instance.error
+        return instance
+    }
+
+    componentDidUpdate(prevProps) {
+        this.updateClientOptions()
+    }
+
+    getAvatarMXC(client) {
+        return client.avatar_url === "disable" ? client.remote_avatar_url : client.avatar_url
+    }
+
+    getAvatarURL(client) {
+        return api.getAvatarURL({
+            id: client.id,
+            avatar_url: this.getAvatarMXC(client),
+        })
+    }
+
+    clientSelectEntry = client => client && {
+        id: client.id,
+        value: client.id,
+        label: (
+            <div className="select-client">
+                {this.getAvatarMXC(client)
+                    ? <img className="avatar" src={this.getAvatarURL(client)} alt=""/>
+                    : <NoAvatarIcon className='avatar'/>}
+                <span className="displayname">{
+                    (client.displayname === "disable"
+                        ? client.remote_displayname
+                        : client.displayname
+                    ) || client.id
+                }</span>
+            </div>
+        ),
+    }
+
+    updateClientOptions() {
+        this.clientOptions = Object.values(this.props.ctx.clients).map(this.clientSelectEntry)
+    }
+
+    save = async () => {
+        this.setState({ saving: true })
+        const resp = await api.putInstance(this.instanceInState, this.props.entry
+            ? this.props.entry.id : undefined)
+        if (resp.id) {
+            if (this.isNew) {
+                this.props.history.push(`/instance/${resp.id}`)
+            } else {
+                if (resp.id !== this.props.entry.id) {
+                    this.props.history.replace(`/instance/${resp.id}`)
+                }
+                this.setState({ saving: false, error: "" })
+            }
+            this.props.onChange(resp)
+        } else {
+            this.setState({ saving: false, error: resp.error })
+        }
+    }
+
+    get selectedClientEntry() {
+        return this.state.primary_user
+            ? this.clientSelectEntry(this.props.ctx.clients[this.state.primary_user])
+            : {}
+    }
+
+    get selectedPluginEntry() {
+        return {
+            id: this.state.type,
+            value: this.state.type,
+            label: this.state.type,
+        }
+    }
+
+    get typeOptions() {
+        return Object.values(this.props.ctx.plugins).map(plugin => plugin && {
+            id: plugin.id,
+            value: plugin.id,
+            label: plugin.id,
+        })
+    }
+
+    get loading() {
+        return this.state.deleting || this.state.saving
+    }
+
+    get isValid() {
+        return this.state.id && this.state.primary_user && this.state.type
+    }
+
+    render() {
+        return <Switch>
+            <Route path="/instance/:id/database" render={this.renderDatabase}/>
+            <Route render={this.renderMain}/>
+        </Switch>
+    }
+
+    renderDatabase = () => <InstanceDatabase instanceID={this.props.entry.id}/>
+
+    renderMain = () => <div className="instance">
+        <PrefTable>
+            <PrefInput rowName="ID" type="text" name="id" value={this.state.id}
+                       placeholder="fancybotinstance" onChange={this.inputChange}
+                       disabled={!this.isNew} fullWidth={true} className="id"/>
+            <PrefSwitch rowName="Enabled"
+                        active={this.state.enabled} origActive={this.props.entry.enabled}
+                        onToggle={enabled => this.setState({ enabled })}/>
+            <PrefSwitch rowName="Running"
+                        active={this.state.started} origActive={this.props.entry.started}
+                        onToggle={started => this.setState({ started })}/>
+            {api.getFeatures().client ? (
+                <PrefSelect rowName="Primary user" options={this.clientOptions}
+                            isSearchable={false} value={this.selectedClientEntry}
+                            origValue={this.props.entry.primary_user}
+                            onChange={({ id }) => this.setState({ primary_user: id })}/>
+            ) : (
+                <PrefInput rowName="Primary user" type="text" name="primary_user"
+                           value={this.state.primary_user} placeholder="@user:example.com"
+                           onChange={this.inputChange}/>
+            )}
+            {api.getFeatures().plugin ? (
+                <PrefSelect rowName="Type" options={this.typeOptions} isSearchable={false}
+                            value={this.selectedPluginEntry} origValue={this.props.entry.type}
+                            onChange={({ id }) => this.setState({ type: id })}/>
+            ) : (
+                <PrefInput rowName="Type" type="text" name="type" value={this.state.type}
+                           placeholder="xyz.maubot.example" onChange={this.inputChange}/>
+            )}
+        </PrefTable>
+        {!this.isNew && Boolean(this.props.entry.base_config) &&
+        <AceEditor mode="yaml" theme="github" onChange={config => this.setState({ config })}
+                   name="config" value={this.state.config}
+                   editorProps={{
+                       fontSize: "10pt",
+                       $blockScrolling: true,
+                   }}/>}
+        <div className="buttons">
+            {!this.isNew && (
+                <button className="delete" onClick={this.delete} disabled={this.loading}>
+                    {this.state.deleting ? <Spinner/> : "Delete"}
+                </button>
+            )}
+            <button className={`save ${this.isValid ? "" : "disabled-bg"}`}
+                    onClick={this.save} disabled={this.loading || !this.isValid}>
+                {this.state.saving ? <Spinner/> : (this.isNew ? "Create" : "Save")}
+            </button>
+        </div>
+        {!this.isNew && <div className="buttons">
+            {this.props.entry.database && (
+                <Link className="button open-database"
+                      to={`/instance/${this.state.id}/database`}>
+                    View database
+                </Link>
+            )}
+            {api.getFeatures().log &&
+            <button className="open-log"
+                    onClick={() => this.props.openLog(`instance.${this.state.id}`)}>
+                View logs
+            </button>}
+        </div>}
+        <div className="error">{this.state.error}</div>
+    </div>
+}
+
+export default withRouter(Instance)

maubot-src/maubot/management/frontend/src/pages/dashboard/InstanceDatabase.js

@@ -0,0 +1,332 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+import { NavLink, Link, withRouter } from "react-router-dom"
+import { ContextMenu, ContextMenuTrigger, MenuItem } from "react-contextmenu"
+import { ReactComponent as ChevronLeft } from "../../res/chevron-left.svg"
+import { ReactComponent as OrderDesc } from "../../res/sort-down.svg"
+import { ReactComponent as OrderAsc } from "../../res/sort-up.svg"
+import api from "../../api"
+import Spinner from "../../components/Spinner"
+
+// eslint-disable-next-line no-extend-native
+Map.prototype.map = function(func) {
+    const res = []
+    for (const [key, value] of this) {
+        res.push(func(value, key, this))
+    }
+    return res
+}
+
+class InstanceDatabase extends Component {
+    constructor(props) {
+        super(props)
+        this.state = {
+            tables: null,
+            header: null,
+            content: null,
+            query: "",
+            selectedTable: null,
+
+            error: null,
+
+            prevQuery: null,
+            statusMsg: null,
+            rowCount: null,
+            insertedPrimaryKey: null,
+        }
+        this.order = new Map()
+    }
+
+    async componentDidMount() {
+        const tables = new Map(Object.entries(await api.getInstanceDatabase(this.props.instanceID)))
+        for (const [name, table] of tables) {
+            table.name = name
+            table.columns = new Map(Object.entries(table.columns))
+            for (const [columnName, column] of table.columns) {
+                column.name = columnName
+                column.sort = null
+            }
+        }
+        this.setState({ tables })
+        this.checkLocationTable()
+    }
+
+    componentDidUpdate(prevProps) {
+        if (this.props.location !== prevProps.location) {
+            this.order = new Map()
+            this.setState({ header: null, content: null })
+            this.checkLocationTable()
+        }
+    }
+
+    checkLocationTable() {
+        const prefix = `/instance/${this.props.instanceID}/database/`
+        if (this.props.location.pathname.startsWith(prefix)) {
+            const table = this.props.location.pathname.substr(prefix.length)
+            this.setState({ selectedTable: table })
+            this.buildSQLQuery(table)
+        }
+    }
+
+    getSortQueryParams(table) {
+        const order = []
+        for (const [column, sort] of Array.from(this.order.entries()).reverse()) {
+            order.push(`order=${column}:${sort}`)
+        }
+        return order
+    }
+
+    buildSQLQuery(table = this.state.selectedTable, resetContent = true) {
+        let query = `SELECT * FROM "${table}"`
+
+        if (this.order.size > 0) {
+            const order = Array.from(this.order.entries()).reverse()
+                .map(([column, sort]) => `${column} ${sort}`)
+            query += ` ORDER BY ${order.join(", ")}`
+        }
+
+        query += " LIMIT 100"
+        this.setState({ query }, () => this.reloadContent(resetContent))
+    }
+
+    reloadContent = async (resetContent = true) => {
+        this.setState({ loading: true })
+        const res = await api.queryInstanceDatabase(this.props.instanceID, this.state.query)
+        this.setState({ loading: false })
+        if (resetContent) {
+            this.setState({
+                prevQuery: null,
+                rowCount: null,
+                insertedPrimaryKey: null,
+                statusMsg: null,
+                error: null,
+            })
+        }
+        if (!res.ok) {
+            this.setState({
+                error: res.error,
+            })
+        } else if (res.rows) {
+            this.setState({
+                header: res.columns,
+                content: res.rows,
+            })
+        } else {
+            this.setState({
+                prevQuery: res.query,
+                rowCount: res.rowcount,
+                insertedPrimaryKey: res.inserted_primary_key,
+                statusMsg: res.status_msg,
+            })
+            this.buildSQLQuery(this.state.selectedTable, false)
+        }
+    }
+
+    toggleSort(column) {
+        const oldSort = this.order.get(column) || "auto"
+        this.order.delete(column)
+        switch (oldSort) {
+        case "auto":
+            this.order.set(column, "DESC")
+            break
+        case "DESC":
+            this.order.set(column, "ASC")
+            break
+        case "ASC":
+        default:
+            break
+        }
+        this.buildSQLQuery()
+    }
+
+    getSortIcon(column) {
+        switch (this.order.get(column)) {
+        case "DESC":
+            return <OrderDesc/>
+        case "ASC":
+            return <OrderAsc/>
+        default:
+            return null
+        }
+    }
+
+    getColumnInfo(columnName) {
+        const table = this.state.tables.get(this.state.selectedTable)
+        if (!table) {
+            return null
+        }
+        const column = table.columns.get(columnName)
+        if (!column) {
+            return null
+        }
+        if (column.primary) {
+            return <span className="meta">&nbsp;(pk)</span>
+        } else if (column.unique) {
+            return <span className="meta">&nbsp;(u)</span>
+        }
+        return null
+    }
+
+    getColumnType(columnName) {
+        const table = this.state.tables.get(this.state.selectedTable)
+        if (!table) {
+            return null
+        }
+        const column = table.columns.get(columnName)
+        if (!column) {
+            return null
+        }
+        return column.type
+    }
+
+    deleteRow = async (_, data) => {
+        const values = this.state.content[data.row]
+        const keys = this.state.header
+        const condition = []
+        for (const [index, key] of Object.entries(keys)) {
+            const val = values[index]
+            condition.push(`${key}='${this.sqlEscape(val.toString())}'`)
+        }
+        const query = `DELETE FROM "${this.state.selectedTable}" WHERE ${condition.join(" AND ")}`
+        const res = await api.queryInstanceDatabase(this.props.instanceID, query)
+        this.setState({
+            prevQuery: `DELETE FROM "${this.state.selectedTable}" ...`,
+            rowCount: res.rowcount,
+        })
+        await this.reloadContent(false)
+    }
+
+    editCell = async (evt, data) => {
+        console.log("Edit", data)
+    }
+
+    collectContextMeta = props => ({
+        row: props.row,
+        col: props.col,
+    })
+
+    // eslint-disable-next-line no-control-regex
+    sqlEscape = str => str.replace(/[\0\x08\x09\x1a\n\r"'\\%]/g, char => {
+        switch (char) {
+        case "\0":
+            return "\\0"
+        case "\x08":
+            return "\\b"
+        case "\x09":
+            return "\\t"
+        case "\x1a":
+            return "\\z"
+        case "\n":
+            return "\\n"
+        case "\r":
+            return "\\r"
+        case "\"":
+        case "'":
+        case "\\":
+        case "%":
+            return "\\" + char
+        default:
+            return char
+        }
+    })
+
+    renderTable = () => <div className="table">
+        {this.state.header ? <>
+            <table>
+                <thead>
+                    <tr>
+                        {this.state.header.map(column => (
+                            <td key={column}>
+                                <span onClick={() => this.toggleSort(column)}
+                                      title={this.getColumnType(column)}>
+                                    <strong>{column}</strong>
+                                    {this.getColumnInfo(column)}
+                                    {this.getSortIcon(column)}
+                                </span>
+                            </td>
+                        ))}
+                    </tr>
+                </thead>
+                <tbody>
+                    {this.state.content.map((row, rowIndex) => (
+                        <tr key={rowIndex}>
+                            {row.map((cell, colIndex) => (
+                                <ContextMenuTrigger key={colIndex} id="database_table_menu"
+                                                    renderTag="td" row={rowIndex} col={colIndex}
+                                                    collect={this.collectContextMeta}>
+                                    {cell}
+                                </ContextMenuTrigger>
+                            ))}
+                        </tr>
+                    ))}
+                </tbody>
+            </table>
+            <ContextMenu id="database_table_menu">
+                <MenuItem onClick={this.deleteRow}>Delete row</MenuItem>
+                <MenuItem disabled onClick={this.editCell}>Edit cell</MenuItem>
+            </ContextMenu>
+        </> : this.state.loading ? <Spinner/> : null}
+    </div>
+
+    renderContent() {
+        return <>
+            <div className="tables">
+                {this.state.tables.map((_, tbl) => (
+                    <NavLink key={tbl} to={`/instance/${this.props.instanceID}/database/${tbl}`}>
+                        {tbl}
+                    </NavLink>
+                ))}
+            </div>
+            <div className="query">
+                <input type="text" value={this.state.query} name="query"
+                       onChange={evt => this.setState({ query: evt.target.value })}/>
+                <button type="submit" onClick={this.reloadContent}>Query</button>
+            </div>
+            {this.state.error && <div className="error">
+                {this.state.error}
+            </div>}
+            {this.state.prevQuery && <div className="prev-query">
+                <p>
+                    Executed <span className="query">{this.state.prevQuery}</span> - {
+                        this.state.statusMsg
+                        || <>affected <strong>{this.state.rowCount} rows</strong>.</>
+                    }
+                </p>
+                {this.state.insertedPrimaryKey && <p className="inserted-primary-key">
+                    Inserted primary key: {this.state.insertedPrimaryKey}
+                </p>}
+            </div>}
+            {this.renderTable()}
+        </>
+    }
+
+    render() {
+        return <div className="instance-database">
+            <div className="topbar">
+                <Link className="topbar" to={`/instance/${this.props.instanceID}`}>
+                    <ChevronLeft/>
+                    Back
+                </Link>
+            </div>
+            {this.state.tables
+                ? this.renderContent()
+                : <Spinner className="maubot-loading"/>}
+        </div>
+    }
+}
+
+export default withRouter(InstanceDatabase)

maubot-src/maubot/management/frontend/src/pages/dashboard/Log.js

@@ -0,0 +1,191 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { PureComponent } from "react"
+import { Link } from "react-router-dom"
+import { JSONTree } from "react-json-tree"
+import api from "../../api"
+import Modal from "./Modal"
+
+class LogEntry extends PureComponent {
+    static contextType = Modal.Context
+
+    renderName() {
+        const line = this.props.line
+        if (line.nameLink) {
+            const modal = this.context
+            return <>
+                <Link to={line.nameLink} onClick={modal.close}>
+                    {line.name}
+                </Link>
+                {line.nameSuffix}
+            </>
+        }
+        return line.name
+    }
+
+    renderContent() {
+        if (this.props.line.matrix_http_request) {
+            const req = this.props.line.matrix_http_request
+
+            return <>
+                {req.method} {req.url || req.path}
+                <div className="content">
+                    {Object.entries(req.content || {}).length > 0
+                    && <JSONTree data={{ content: req.content }} hideRoot={true}/>}
+                </div>
+            </>
+        }
+        return this.props.line.msg
+    }
+
+    onClickOpen(path, line) {
+        return () => {
+            if (api.debugOpenFileEnabled()) {
+                api.debugOpenFile(path, line)
+            }
+            return false
+        }
+    }
+
+    renderTimeTitle() {
+        return this.props.line.time.toDateString()
+    }
+
+    renderTime() {
+        return <a className="time" title={this.renderTimeTitle()}
+                  href={`file:///${this.props.line.pathname}:${this.props.line.lineno}`}
+                  onClick={this.onClickOpen(this.props.line.pathname, this.props.line.lineno)}>
+            {this.props.line.time.toLocaleTimeString("en-GB")}
+        </a>
+    }
+
+    renderLevelName() {
+        return <span className="level">
+            {this.props.line.levelname}
+        </span>
+    }
+
+    get unfocused() {
+        return this.props.focus && this.props.line.name !== this.props.focus
+            ? "unfocused"
+            : ""
+    }
+
+    renderRow(content) {
+        return (
+            <div className={`row ${this.props.line.levelname.toLowerCase()} ${this.unfocused}`}>
+                {this.renderTime()}
+                {this.renderLevelName()}
+                <span className="logger">{this.renderName()}</span>
+                <span className="text">{content}</span>
+            </div>
+        )
+    }
+
+    renderExceptionInfo() {
+        if (!api.debugOpenFileEnabled()) {
+            return this.props.line.exc_info
+        }
+        const fileLinks = []
+        let str = this.props.line.exc_info.replace(
+            /File "(.+)", line ([0-9]+), in (.+)/g,
+            (_, file, line, method) => {
+                fileLinks.push(
+                    <a href={`file:///${file}:${line}`} onClick={this.onClickOpen(file, line)}>
+                        File "{file}", line {line}, in {method}
+                    </a>,
+                )
+                return "||EDGE||"
+            })
+        fileLinks.reverse()
+
+        const result = []
+        let key = 0
+        for (const part of str.split("||EDGE||")) {
+            result.push(<React.Fragment key={key++}>
+                {part}
+                {fileLinks.pop()}
+            </React.Fragment>)
+        }
+        return result
+    }
+
+    render() {
+        return <>
+            {this.renderRow(this.renderContent())}
+            {this.props.line.exc_info && this.renderRow(this.renderExceptionInfo())}
+        </>
+    }
+}
+
+class Log extends PureComponent {
+    constructor(props) {
+        super(props)
+
+        this.linesRef = React.createRef()
+        this.linesBottomRef = React.createRef()
+    }
+
+    getSnapshotBeforeUpdate() {
+        if (this.linesRef.current && this.linesBottomRef.current) {
+            return Log.isVisible(this.linesRef.current, this.linesBottomRef.current)
+        }
+        return false
+    }
+
+
+    componentDidUpdate(_1, _2, wasVisible) {
+        if (wasVisible) {
+            Log.scrollParentToChild(this.linesRef.current, this.linesBottomRef.current)
+        }
+    }
+
+    componentDidMount() {
+        if (this.linesRef.current && this.linesBottomRef.current) {
+            Log.scrollParentToChild(this.linesRef.current, this.linesBottomRef.current)
+        }
+    }
+
+    static scrollParentToChild(parent, child) {
+        const parentRect = parent.getBoundingClientRect()
+        const childRect = child.getBoundingClientRect()
+
+        if (!Log.isVisible(parent, child)) {
+            parent.scrollBy({ top: (childRect.top + parent.scrollTop) - parentRect.top })
+        }
+    }
+
+    static isVisible(parent, child) {
+        const parentRect = parent.getBoundingClientRect()
+        const childRect = child.getBoundingClientRect()
+        return (childRect.top >= parentRect.top)
+            && (childRect.top <= parentRect.top + parent.clientHeight)
+    }
+
+    render() {
+        return (
+            <div className="log" ref={this.linesRef}>
+                <div className="lines">
+                    {this.props.lines.map(data => <LogEntry key={data.id} line={data}
+                                                            focus={this.props.focus}/>)}
+                </div>
+                <div ref={this.linesBottomRef}/>
+            </div>
+        )
+    }
+}
+
+export default Log

maubot-src/maubot/management/frontend/src/pages/dashboard/Modal.js

@@ -0,0 +1,52 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component, createContext } from "react"
+
+const rem = 16
+
+class Modal extends Component {
+    static Context = createContext(null)
+
+    constructor(props) {
+        super(props)
+        this.state = {
+            open: false,
+        }
+        this.wrapper = { clientWidth: 9001 }
+    }
+
+    open = () => this.setState({ open: true })
+    close = () => this.setState({ open: false })
+    isOpen = () => this.state.open
+
+    render() {
+        return this.state.open && (
+            <div className="modal-wrapper-wrapper" ref={ref => this.wrapper = ref}
+                 onClick={() => this.wrapper.clientWidth > 45 * rem && this.close()}>
+                <div className="modal-wrapper" onClick={evt => evt.stopPropagation()}>
+                    <button className="close" onClick={this.close}>Close</button>
+                    <div className="modal">
+                        <Modal.Context.Provider value={this}>
+                            {this.props.children}
+                        </Modal.Context.Provider>
+                    </div>
+                </div>
+            </div>
+        )
+    }
+}
+
+export default Modal

maubot-src/maubot/management/frontend/src/pages/dashboard/Plugin.js

@@ -0,0 +1,104 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React from "react"
+import { NavLink, withRouter } from "react-router-dom"
+import { ReactComponent as ChevronRight } from "../../res/chevron-right.svg"
+import { ReactComponent as UploadButton } from "../../res/upload.svg"
+import PrefTable, { PrefInput } from "../../components/PreferenceTable"
+import Spinner from "../../components/Spinner"
+import api from "../../api"
+import BaseMainView from "./BaseMainView"
+
+const PluginListEntry = ({ entry }) => (
+    <NavLink className="plugin entry" to={`/plugin/${entry.id}`}>
+        <span className="id">{entry.id}</span>
+        <ChevronRight className='chevron'/>
+    </NavLink>
+)
+
+
+class Plugin extends BaseMainView {
+    static ListEntry = PluginListEntry
+
+    constructor(props) {
+        super(props)
+        this.deleteFunc = api.deletePlugin
+    }
+
+    get initialState() {
+        return {
+            id: "",
+            version: "",
+
+            instances: [],
+
+            uploading: false,
+            deleting: false,
+            error: "",
+        }
+    }
+
+    upload = async event => {
+        const file = event.target.files[0]
+        this.setState({
+            uploadingAvatar: true,
+        })
+        const data = await this.readFile(file)
+        const resp = await api.uploadPlugin(data, this.state.id)
+        if (resp.id) {
+            if (this.isNew) {
+                this.props.history.push(`/plugin/${resp.id}`)
+            } else {
+                this.setState({ saving: false, error: "" })
+            }
+            this.props.onChange(resp)
+        } else {
+            this.setState({ saving: false, error: resp.error })
+        }
+    }
+
+    render() {
+        return <div className="plugin">
+            {!this.isNew && <PrefTable>
+                <PrefInput rowName="ID" type="text" value={this.state.id} disabled={true}
+                           className="id"/>
+                <PrefInput rowName="Version" type="text" value={this.state.version}
+                           disabled={true}/>
+            </PrefTable>}
+            {api.getFeatures().plugin_upload &&
+            <div className={`upload-box ${this.state.uploading ? "uploading" : ""}`}>
+                <UploadButton className="upload"/>
+                <input className="file-selector" type="file" accept="application/zip+mbp"
+                       onChange={this.upload} disabled={this.state.uploading || this.state.deleting}
+                       onDragEnter={evt => evt.target.parentElement.classList.add("drag")}
+                       onDragLeave={evt => evt.target.parentElement.classList.remove("drag")}/>
+                {this.state.uploading && <Spinner/>}
+            </div>}
+            {!this.isNew && <div className="buttons">
+                <button className={`delete ${this.hasInstances ? "disabled-bg" : ""}`}
+                        onClick={this.delete} disabled={this.loading || this.hasInstances}
+                        title={this.hasInstances ? "Can't delete plugin that is in use" : ""}>
+                    {this.state.deleting ? <Spinner/> : "Delete"}
+                </button>
+            </div>}
+            {this.renderLogButton("loader.zip")}
+            <div className="error">{this.state.error}</div>
+            {this.renderInstances()}
+        </div>
+    }
+}
+
+export default withRouter(Plugin)

maubot-src/maubot/management/frontend/src/pages/dashboard/index.js

@@ -0,0 +1,272 @@
+// maubot - A plugin-based Matrix bot system.
+// Copyright (C) 2022 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import React, { Component } from "react"
+import { Route, Switch, Link, withRouter } from "react-router-dom"
+import api from "../../api"
+import { ReactComponent as Plus } from "../../res/plus.svg"
+import Instance from "./Instance"
+import Client from "./Client"
+import Plugin from "./Plugin"
+import Home from "./Home"
+import Log from "./Log"
+import Modal from "./Modal"
+
+class Dashboard extends Component {
+    constructor(props) {
+        super(props)
+        this.state = {
+            instances: {},
+            clients: {},
+            plugins: {},
+            homeserversByName: {},
+            homeserversByURL: {},
+            sidebarOpen: false,
+            modalOpen: false,
+            logFocus: null,
+            logLines: [],
+        }
+        this.logModal = {
+            open: () => undefined,
+            isOpen: () => false,
+        }
+        window.maubot = this
+    }
+
+    componentDidUpdate(prevProps) {
+        if (this.props.location !== prevProps.location) {
+            this.setState({ sidebarOpen: false })
+        }
+    }
+
+    async componentDidMount() {
+        const [instanceList, clientList, pluginList, homeservers] = await Promise.all([
+            api.getInstances(), api.getClients(), api.getPlugins(), api.getClientAuthServers(),
+            api.updateDebugOpenFileEnabled()])
+        const instances = {}
+        if (api.getFeatures().instance) {
+            for (const instance of instanceList) {
+                instances[instance.id] = instance
+            }
+        }
+        const clients = {}
+        if (api.getFeatures().client) {
+            for (const client of clientList) {
+                clients[client.id] = client
+            }
+        }
+        const plugins = {}
+        if (api.getFeatures().plugin) {
+            for (const plugin of pluginList) {
+                plugins[plugin.id] = plugin
+            }
+        }
+        const homeserversByName = homeservers
+        const homeserversByURL = {}
+        if (api.getFeatures().client_auth) {
+            for (const [key, value] of Object.entries(homeservers)) {
+                homeserversByURL[value] = key
+            }
+        }
+        this.setState({ instances, clients, plugins, homeserversByName, homeserversByURL })
+
+        await this.enableLogs()
+    }
+
+    async enableLogs() {
+        if (!api.getFeatures().log) {
+            return
+        }
+
+        const logs = await api.openLogSocket()
+
+        const processEntry = (entry) => {
+            entry.time = new Date(entry.time)
+            entry.nameSuffix = ""
+            if (entry.name.startsWith("maubot.client.")) {
+                if (entry.name.endsWith(".crypto")) {
+                    entry.name = entry.name.slice(0, -".crypto".length)
+                    entry.nameSuffix = "/crypto"
+                }
+                entry.name = `client/${entry.name.slice("maubot.client.".length)}`
+                entry.nameLink = `/${entry.name}`
+            } else if (entry.name.startsWith("maubot.instance.")) {
+                entry.name = `instance/${entry.name.slice("maubot.instance.".length)}`
+                entry.nameLink = `/${entry.name}`
+            } else if (entry.name.startsWith("maubot.instance_db.")) {
+                entry.nameSuffix = "/db"
+                entry.name = `instance/${entry.name.slice("maubot.instance_db.".length)}`
+                entry.nameLink = `/${entry.name}`
+            }
+        }
+
+        logs.onHistory = history => {
+            for (const data of history) {
+                processEntry(data)
+            }
+            this.setState({
+                logLines: history,
+            })
+        }
+        logs.onLog = data => {
+            processEntry(data)
+            this.setState({
+                logLines: this.state.logLines.concat(data),
+            })
+        }
+    }
+
+    renderList(field, type) {
+        return this.state[field] && Object.values(this.state[field]).map(entry =>
+            React.createElement(type, { key: entry.id, entry }))
+    }
+
+    delete(stateField, id) {
+        const data = Object.assign({}, this.state[stateField])
+        delete data[id]
+        this.setState({ [stateField]: data })
+    }
+
+    add(stateField, entry, oldID = undefined) {
+        const data = Object.assign({}, this.state[stateField])
+        if (oldID && oldID !== entry.id) {
+            delete data[oldID]
+        }
+        data[entry.id] = entry
+        this.setState({ [stateField]: data })
+    }
+
+    renderView(field, type, id) {
+        const typeName = field.slice(0, -1)
+        if (!api.getFeatures()[typeName]) {
+            return this.renderDisabled(typeName)
+        }
+        const entry = this.state[field][id]
+        if (!entry) {
+            return this.renderNotFound(typeName)
+        }
+        return React.createElement(type, {
+            entry,
+            onDelete: () => this.delete(field, id),
+            onChange: newEntry => this.add(field, newEntry, id),
+            openLog: this.openLog,
+            ctx: this.state,
+        })
+    }
+
+    openLog = filter => {
+        this.setState({
+            logFocus: typeof filter === "string" ? filter : null,
+        })
+        this.logModal.open()
+    }
+
+    renderNotFound = (thing = "path") => (
+        <div className="not-found">
+            Oops! I'm afraid that {thing} couldn't be found.
+        </div>
+    )
+
+    renderDisabled = (thing = "path") => (
+        <div className="not-found">
+            The {thing} API has been disabled in the maubot config.
+        </div>
+    )
+
+    renderMain() {
+        return <div className={`dashboard ${this.state.sidebarOpen ? "sidebar-open" : ""}`}>
+            <Link to="/" className="title">
+                <img src="favicon.png" alt=""/>
+                Maubot Manager
+            </Link>
+            <div className="user">
+                <span>{localStorage.username}</span>
+            </div>
+
+            <nav className="sidebar">
+                <div className="buttons">
+                    {api.getFeatures().log && <button className="open-log" onClick={this.openLog}>
+                        <span>View logs</span>
+                    </button>}
+                </div>
+                {api.getFeatures().instance && <div className="instances list">
+                    <div className="title">
+                        <h2>Instances</h2>
+                        <Link to="/new/instance"><Plus/></Link>
+                    </div>
+                    {this.renderList("instances", Instance.ListEntry)}
+                </div>}
+                {api.getFeatures().client && <div className="clients list">
+                    <div className="title">
+                        <h2>Clients</h2>
+                        <Link to="/new/client"><Plus/></Link>
+                    </div>
+                    {this.renderList("clients", Client.ListEntry)}
+                </div>}
+                {api.getFeatures().plugin && <div className="plugins list">
+                    <div className="title">
+                        <h2>Plugins</h2>
+                        {api.getFeatures().plugin_upload && <Link to="/new/plugin"><Plus/></Link>}
+                    </div>
+                    {this.renderList("plugins", Plugin.ListEntry)}
+                </div>}
+            </nav>
+
+            <div className="topbar"
+                 onClick={evt => this.setState({ sidebarOpen: !this.state.sidebarOpen })}>
+                <div className={`hamburger ${this.state.sidebarOpen ? "active" : ""}`}>
+                    <span/><span/><span/>
+                </div>
+            </div>
+
+            <main className="view">
+                <Switch>
+                    <Route path="/" exact render={() => <Home openLog={this.openLog}/>}/>
+                    <Route path="/new/instance" render={() =>
+                        <Instance onChange={newEntry => this.add("instances", newEntry)}
+                                  entry={{}} ctx={this.state}/>}/>
+                    <Route path="/new/client" render={() =>
+                        <Client onChange={newEntry => this.add("clients", newEntry)}
+                                entry={{}} ctx={this.state}/>}/>
+                    <Route path="/new/plugin" render={() =>
+                        <Plugin onChange={newEntry => this.add("plugins", newEntry)}
+                                entry={{}} ctx={this.state}/>}/>
+                    <Route path="/instance/:id" render={({ match }) =>
+                        this.renderView("instances", Instance, match.params.id)}/>
+                    <Route path="/client/:id" render={({ match }) =>
+                        this.renderView("clients", Client, match.params.id)}/>
+                    <Route path="/plugin/:id" render={({ match }) =>
+                        this.renderView("plugins", Plugin, match.params.id)}/>
+                    <Route render={() => this.renderNotFound()}/>
+                </Switch>
+            </main>
+        </div>
+    }
+
+    renderModal() {
+        return <Modal ref={ref => this.logModal = ref}>
+            <Log lines={this.state.logLines} focus={this.state.logFocus}/>
+        </Modal>
+    }
+
+    render() {
+        return <>
+            {this.renderMain()}
+            {this.renderModal()}
+        </>
+    }
+}
+
+export default withRouter(Dashboard)

maubot-src/maubot/management/frontend/src/res/bot.svg

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1"  width="24" height="24" viewBox="0 0 24 24">
+   <path fill="#000000" d="M12,2A2,2 0 0,1 14,4C14,4.74 13.6,5.39 13,5.73V7H14A7,7 0 0,1 21,14H22A1,1 0 0,1 23,15V18A1,1 0 0,1 22,19H21V20A2,2 0 0,1 19,22H5A2,2 0 0,1 3,20V19H2A1,1 0 0,1 1,18V15A1,1 0 0,1 2,14H3A7,7 0 0,1 10,7H11V5.73C10.4,5.39 10,4.74 10,4A2,2 0 0,1 12,2M7.5,13A2.5,2.5 0 0,0 5,15.5A2.5,2.5 0 0,0 7.5,18A2.5,2.5 0 0,0 10,15.5A2.5,2.5 0 0,0 7.5,13M16.5,13A2.5,2.5 0 0,0 14,15.5A2.5,2.5 0 0,0 16.5,18A2.5,2.5 0 0,0 19,15.5A2.5,2.5 0 0,0 16.5,13Z" />
+</svg>

maubot-src/maubot/management/frontend/src/res/chevron-left.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"><path d="M15.41 7.41L14 6l-6 6 6 6 1.41-1.41L10.83 12z"/><path d="M0 0h24v24H0z" fill="none"/></svg>

maubot-src/maubot/management/frontend/src/res/chevron-right.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"/><path d="M0 0h24v24H0z" fill="none"/></svg>

maubot-src/maubot/management/frontend/src/res/plus.svg

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1"  width="24" height="24" viewBox="0 0 24 24">
+   <path fill="#000000" d="M17,13H13V17H11V13H7V11H11V7H13V11H17M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2Z" />
+</svg>

maubot-src/maubot/management/frontend/src/res/sort-down.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"><path d="M7 10l5 5 5-5z"/><path d="M0 0h24v24H0z" fill="none"/></svg>