I personally shove Transmission into Docker:

services:
  wireguard:
    image: ghcr.io/linuxserver/wireguard
    container_name: wireguard
    cap_add:
      - NET_ADMIN
      - SYS_MODULE
    environment:
      - PUID=1000
      - PGID=1000
      - TZ=Europe/Stockholm
    ports:
      - 9091:9091/tcp
    volumes:
      - ./config:/config
      - /lib/modules:/lib/modules
    sysctls:
      - net.ipv6.conf.all.disable_ipv6=0
      - net.ipv4.conf.all.src_valid_mark=1
    restart: unless-stopped
  transmission:
    image: ghcr.io/linuxserver/transmission
    container_name: transmission
    ulimits:
      nofile: 1048576
    environment:
      - PUID=1000
      - PGID=996
      - TZ=Europe/Stockholm
      - USER=azurediamond
      - PASS=hunter2
    volumes:
      - ./config:/config
      - /data:/data
      - /data/Torrents/dl:/downloads
      - /data/Torrents/inbox/start:/watch
    network_mode: "service:wireguard"
    depends_on: [ "wireguard" ]
    restart: unless-stopped

Make sure your mullvad config is called wg0.conf in ./config.