Self-Hosted Mail Server
A to Z Guide

You are a Linux sysadmin assistant. Audit this Ubuntu VPS and
determine if it's ready for a self-hosted email server (Postfix
SMTP + Dovecot IMAP). Do not install anything.

Network & IP: public IP, outbound port 25/587/465/993/143 test,
PTR/rDNS lookup.

DNS: hostname -f, /etc/hosts, check MX/SPF/DKIM/DMARC records.

IP Reputation: check against Spamhaus ZEN, SpamCop, Barracuda,
SORBS via MXToolbox.

System: OS, disk, RAM, swap, UFW/iptables, any existing mail
packages (postfix/dovecot/exim/sendmail), local DNS resolver.

SSL/TLS: certbot installed? /etc/letsencrypt/ exists?

Verdict: Ready / Needs fixes / Blocked (list specific blockers).

telnet mail.yourdomain.com 25
# Expected: 220 mail.yourdomain.com ESMTP Postfix (Ubuntu)

dig -x YOUR_IP +short
# Expected: mail.yourdomain.com.

dig MX yourdomain.com +short
# Expected: 10 mail.yourdomain.com.

dig A mail.yourdomain.com +short
# Expected: YOUR_IP (NOT 104.x or 172.x Cloudflare IPs)

dig TXT yourdomain.com +short
# Expected: "v=spf1 mx -all"

dig TXT _dmarc.yourdomain.com +short
# Expected: "v=DMARC1; p=none; ..."

sudo hostnamectl set-hostname mail.yourdomain.com

# Replace this:
127.0.1.1   vmi3038401.contaboserver.net vmi3038401

# With this:
127.0.1.1   mail.yourdomain.com mail

hostname -f
# Expected: mail.yourdomain.com
# Note: shell prompt shows user@mail:~$ — that's normal, short hostname only

sudo fallocate -l 2G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
free -h

sudo ufw allow 22/tcp     # SSH
sudo ufw allow 25/tcp     # SMTP inbound
sudo ufw allow 465/tcp    # Submissions SSL wrapper
sudo ufw allow 587/tcp    # Submission STARTTLS
sudo ufw allow 143/tcp    # IMAP
sudo ufw allow 993/tcp    # IMAPS
sudo ufw allow 80/tcp
sudo ufw allow 443/tcp
sudo ufw enable
sudo ufw status verbose

sudo apt install certbot python3-certbot-dns-cloudflare -y

sudo mkdir -p /etc/letsencrypt/cloudflare
sudo nano /etc/letsencrypt/cloudflare/yourdomain.ini

dns_cloudflare_api_token = YOUR_TOKEN_HERE

sudo chmod 600 /etc/letsencrypt/cloudflare/yourdomain.ini

sudo certbot certonly \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare/yourdomain.ini \
  -d mail.yourdomain.com \
  --email [email protected] \
  --agree-tos \
  --no-eff-email

sudo apt update
sudo apt install postfix -y
# Do NOT install postfix-mysql or postfix-pgsql
# Flat files are simpler with zero database dependencies

# Debian defaults
smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
biff = no
append_dot_mydomain = no
readme_directory = no
compatibility_level = 3.6

# Identity
myhostname = mail.yourdomain.com
mydomain = yourdomain.com
myorigin = $mydomain

# Network — use ipv4 only to avoid IPv6 SPF failures
inet_interfaces = all
inet_protocols = ipv4
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128

# yourdomain.com intentionally OMITTED from mydestination
mydestination = $myhostname, localhost.$mydomain, localhost
alias_maps = hash:/etc/aliases
alias_database = hash:/etc/aliases

# Virtual mailboxes (flat file, no database)
virtual_mailbox_domains = yourdomain.com
virtual_mailbox_base = /var/vmail
virtual_mailbox_maps = hash:/etc/postfix/vmailbox
virtual_alias_maps = hash:/etc/postfix/virtual
virtual_uid_maps = static:5000
virtual_gid_maps = static:5000
virtual_mailbox_limit = 0

# TLS
smtpd_tls_cert_file = /etc/letsencrypt/live/mail.yourdomain.com/fullchain.pem
smtpd_tls_key_file = /etc/letsencrypt/live/mail.yourdomain.com/privkey.pem
smtpd_tls_security_level = may
smtp_tls_CApath = /etc/ssl/certs
smtp_tls_security_level = may
smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
smtpd_tls_loglevel = 1

# SMTP auth via Dovecot SASL
smtpd_sasl_type = dovecot
smtpd_sasl_path = private/auth
smtpd_sasl_auth_enable = yes

# Restrictions
smtpd_relay_restrictions = permit_mynetworks, permit_sasl_authenticated, defer_unauth_destination
smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination
disable_vrfy_command = yes
smtpd_helo_required = yes

# Limits
mailbox_size_limit = 0
message_size_limit = 52428800
recipient_delimiter = +
relayhost =

# DKIM milter (inet socket — unix socket fails inside chroot)
milter_default_action = accept
milter_protocol = 6
smtpd_milters = inet:localhost:12301
non_smtpd_milters = inet:localhost:12301

sudo groupadd -g 5000 vmail
sudo useradd -g vmail -u 5000 vmail -d /var/vmail -m
sudo mkdir -p /var/vmail
sudo chown vmail:vmail /var/vmail
sudo chmod 700 /var/vmail

[email protected]    yourdomain.com/user/

sudo touch /etc/postfix/virtual
sudo postmap /etc/postfix/vmailbox
sudo postmap /etc/postfix/virtual

submission inet n       -       y       -       -       smtpd
  -o syslog_name=postfix/submission
  -o smtpd_tls_security_level=encrypt
  -o smtpd_sasl_auth_enable=yes
  -o smtpd_tls_auth_only=yes
  -o smtpd_reject_unlisted_recipient=no
  -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject
  -o smtpd_milters=inet:localhost:12301
  -o milter_macro_daemon_name=ORIGINATING

submissions     inet  n       -       y       -       -       smtpd
  -o syslog_name=postfix/submissions
  -o smtpd_tls_wrappermode=yes
  -o smtpd_sasl_auth_enable=yes
  -o smtpd_reject_unlisted_recipient=no
  -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject
  -o smtpd_milters=inet:localhost:12301
  -o milter_macro_daemon_name=ORIGINATING

sudo postfix check          # silence = good
sudo postfix reload
sudo ss -tlnp | grep master
# Expected: ports 25, 465, 587 all LISTEN

sudo apt install dovecot-core dovecot-imapd -y

protocols = imap

mail_location = maildir:/var/vmail/%d/%n/
# %d = domain, %n = user
mail_privileged_group = vmail

disable_plaintext_auth = yes
auth_mechanisms = plain login

# Comment out system auth, enable passwd-file
#!include auth-system.conf.ext
!include auth-passwdfile.conf.ext

passdb {
  driver = passwd-file
  args = scheme=SHA512-CRYPT username_format=%u /etc/dovecot/users
}

userdb {
  driver = static
  args = uid=vmail gid=vmail home=/var/vmail/%d/%n/
}

ssl = required
ssl_cert = </etc/letsencrypt/live/mail.yourdomain.com/fullchain.pem
ssl_key = </etc/letsencrypt/live/mail.yourdomain.com/privkey.pem
ssl_min_protocol = TLSv1.2

service auth {
  unix_listener /var/spool/postfix/private/auth {
    mode = 0660
    user = postfix
    group = postfix
  }
}

sudo touch /etc/dovecot/users
sudo chmod 640 /etc/dovecot/users
sudo chown root:dovecot /etc/dovecot/users

# Generate password hash
sudo doveadm pw -s SHA512-CRYPT
# Outputs: {SHA512-CRYPT}$6$abc...

[email protected]:{SHA512-CRYPT}$6$yourhashhere

sudo systemctl restart dovecot
sudo doveadm auth test [email protected]
# Expected:
# passdb: [email protected] auth succeeded
# userdb: [email protected] auth succeeded

sudo apt install opendkim opendkim-tools -y

Syslog                  yes
SyslogSuccess           yes
LogWhy                  yes

Canonicalization        relaxed/simple
Mode                    sv
SubDomains              no
OversignHeaders         From

KeyTable                /etc/opendkim/KeyTable
SigningTable            refile:/etc/opendkim/SigningTable
ExternalIgnoreList      /etc/opendkim/TrustedHosts
InternalHosts           /etc/opendkim/TrustedHosts

Socket                  inet:12301@localhost
PidFile                 /run/opendkim/opendkim.pid
UserID                  opendkim
UMask                   007

TrustAnchorFile         /usr/share/dns/root.key
TemporaryDirectory      /var/tmp

sudo mkdir -p /etc/opendkim/keys/yourdomain.com
sudo opendkim-genkey -b 2048 -d yourdomain.com \
  -D /etc/opendkim/keys/yourdomain.com -s mail -v
sudo chown opendkim:root /etc/opendkim/keys/yourdomain.com/mail.private
sudo chmod 600 /etc/opendkim/keys/yourdomain.com/mail.private

mail._domainkey.yourdomain.com    yourdomain.com:mail:/etc/opendkim/keys/yourdomain.com/mail.private

*@yourdomain.com    mail._domainkey.yourdomain.com

127.0.0.1
::1
localhost
yourdomain.com
mail.yourdomain.com

sudo systemctl enable opendkim
sudo systemctl restart opendkim
sudo ss -tlnp | grep 12301
# Expected: LISTEN on 127.0.0.1:12301

sudo cat /etc/opendkim/keys/yourdomain.com/mail.txt
# Outputs multi-line quoted strings — concatenate ALL parts
# into a single string with no spaces between segments

sudo opendkim-testkey -d yourdomain.com -s mail -vvv
# "key not secure" = normal (no DNSSEC) — not a problem
# "key OK" = this is what matters

sudo apt install rsyslog -y
sudo systemctl enable rsyslog
sudo systemctl start rsyslog

# If /var/log/mail.log still doesn't exist:
sudo touch /var/log/mail.log
sudo chown syslog:adm /var/log/mail.log
sudo chmod 640 /var/log/mail.log
sudo systemctl restart rsyslog
sudo systemctl restart postfix

sudo tail -f /var/log/mail.log

# Or via journalctl:
sudo journalctl -u postfix -f
sudo journalctl -u dovecot -f
sudo journalctl -u opendkim -f

sudo apt install swaks -y

swaks \
  --to [email protected] \
  --from [email protected] \
  --server 127.0.0.1 \
  --port 25
# Check: /var/vmail/yourdomain.com/user/new/

swaks \
  --to [email protected] \
  --from [email protected] \
  --server mail.yourdomain.com \
  --port 587 \
  --tls \
  --auth LOGIN \
  --auth-user [email protected] \
  --auth-password 'your_password'
# Note: the flag is --tls, NOT --starttls

swaks \
  --to [email protected] \
  --from [email protected] \
  --server mail.yourdomain.com \
  --port 465 \
  --tlsc \
  --auth LOGIN \
  --auth-user [email protected] \
  --auth-password 'your_password'
# --tlsc = TLS-on-connect (wrapper mode), required for 465

curl -v \
  --url "imaps://mail.yourdomain.com:993/INBOX" \
  --user "[email protected]:your_password"

Authentication-Results: mx.google.com;
       dkim=pass [email protected] header.s=mail
       spf=pass (google.com: domain of [email protected] designates
              YOUR_IP as permitted sender)
       dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=yourdomain.com

sudo nano /etc/letsencrypt/renewal-hooks/deploy/reload-mail.sh

#!/bin/bash
systemctl reload postfix
systemctl reload dovecot

sudo chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-mail.sh

sudo apt install fail2ban -y

[postfix]
enabled = true
port = smtp,465,submission
logpath = /var/log/mail.log

[postfix-sasl]
enabled = true
port = smtp,465,submission,imap,imaps,pop3,pop3s
logpath = /var/log/mail.log

[dovecot]
enabled = true
port = pop3,pop3s,imap,imaps,submission,465,sieve
logpath = /var/log/mail.log

sudo systemctl restart fail2ban
sudo fail2ban-client status

v=DMARC1; p=quarantine; rua=mailto:[email protected]; pct=100
# Eventually: p=reject

sudo apt install rspamd -y
# Configure as milter alongside OpenDKIM
# Skip for personal use — rspamd is non-trivial to tune