Linux Server Hardening & Sysadmin

sudo apt update && sudo apt full-upgrade -y
sudo apt autoremove --purge -y

sudo dnf upgrade --refresh -y

sudo adduser deploy                 # interactive: sets password
sudo usermod -aG sudo deploy        # 'sudo' group grants admin
# verify:
groups deploy

sudo hostnamectl set-hostname web-01
sudo timedatectl set-timezone UTC          # UTC on servers avoids DST bugs
sudo timedatectl set-ntp true              # enable systemd-timesyncd
timedatectl status                         # confirm "System clock synchronized: yes"

ssh-keygen -t ed25519 -a 100 -C "deploy@web-01 2026-06"
# -a 100 = 100 KDF rounds, slows brute-force of the passphrase
# For hosts stuck on very old OpenSSH (<6.5), fall back to:
ssh-keygen -t rsa -b 4096 -C "deploy@web-01 2026-06"

eval "$(ssh-agent -s)"
ssh-add ~/.ssh/id_ed25519          # macOS: ssh-add --apple-use-keychain

ssh-copy-id deploy@SERVER_IP        # prompts for password one last time
ssh deploy@SERVER_IP                # should log in with NO password prompt

cat ~/.ssh/id_ed25519.pub | ssh deploy@SERVER_IP \
  "mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"

sudo tee /etc/ssh/sshd_config.d/99-hardening.conf >/dev/null <<'EOF'
# --- Authentication ---
PermitRootLogin no
PasswordAuthentication no
KbdInteractiveAuthentication no
PubkeyAuthentication yes
AuthenticationMethods publickey
PermitEmptyPasswords no
MaxAuthTries 3
LoginGraceTime 20
MaxSessions 4

# --- Limit who can log in ---
AllowUsers deploy

# --- Reduce attack surface ---
X11Forwarding no
AllowAgentForwarding no
AllowTcpForwarding no
ClientAliveInterval 300
ClientAliveCountMax 2
EOF

sudo sshd -t && echo "config OK"     # aborts here if syntax is wrong
sudo systemctl reload ssh             # 'reload' keeps your current session alive

sudo apt install -y libpam-google-authenticator
google-authenticator      # run as YOUR user; scan QR, save scratch codes
# /etc/pam.d/sshd : add ->  auth required pam_google_authenticator.so
# /etc/ssh/sshd_config.d/99-hardening.conf:
#   AuthenticationMethods publickey,keyboard-interactive
#   KbdInteractiveAuthentication yes
sudo systemctl reload ssh

sudo apt install -y ufw
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw limit OpenSSH          # 'limit' = rate-cap brute force (6 hits/30s -> deny)
sudo ufw allow 80/tcp           # http  (omit if no web server)
sudo ufw allow 443/tcp          # https
sudo ufw enable                 # answer 'y' — SSH already allowed, so safe
sudo ufw status verbose

sudo ufw allow from 203.0.113.10 to any port 22 proto tcp
sudo ufw status numbered         # list with rule numbers
sudo ufw delete 3                # remove rule #3

# from your laptop, NOT the server:
nmap -Pn -p 22,80,443,5432,3306,6379 SERVER_IP
# anything OPEN that you didn't intend is a finding

sudo apt install -y fail2ban

sudo tee /etc/fail2ban/jail.local >/dev/null <<'EOF'
[DEFAULT]
bantime  = 1h
findtime = 10m
maxretry = 5
# never ban yourself — add your office/home IP:
ignoreip = 127.0.0.1/8 ::1 203.0.113.10
# escalate repeat offenders:
bantime.increment = true
bantime.factor    = 2
bantime.maxtime   = 1w

[sshd]
enabled = true
EOF

sudo systemctl enable --now fail2ban
sudo fail2ban-client status            # lists active jails
sudo fail2ban-client status sshd       # banned IPs, total failures

sudo fail2ban-client set sshd unbanip 203.0.113.10

sudo apt install -y unattended-upgrades
sudo dpkg-reconfigure -plow unattended-upgrades   # answer "Yes"

sudo tee /etc/apt/apt.conf.d/20auto-upgrades >/dev/null <<'EOF'
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
APT::Periodic::AutocleanInterval "7";
EOF

# Uncomment to auto-reboot when a patch needs it (e.g. kernel):
Unattended-Upgrade::Automatic-Reboot "true";
Unattended-Upgrade::Automatic-Reboot-Time "03:30";
# Email yourself on action (needs a configured MTA):
Unattended-Upgrade::Mail "[email protected]";
Unattended-Upgrade::MailReport "on-change";

sudo unattended-upgrade --dry-run --debug | tail -n 20

sudo visudo -f /etc/sudoers.d/deploy
# put a single line, then save:
deploy ALL=(ALL:ALL) ALL
# validate the whole tree:
sudo visudo -c

getent group sudo          # Debian/Ubuntu admins (wheel on RHEL)
sudo grep -RvE '^\s*#|^\s*$' /etc/sudoers /etc/sudoers.d/   # all live rules
awk -F: '$3==0 {print $1}' /etc/passwd                      # UID 0 = root; should be ONLY 'root'

sudo passwd -l olduser                 # lock password
sudo usermod -s /usr/sbin/nologin olduser   # deny interactive shell
# find accounts WITH a login shell:
getent passwd | awk -F: '$7 ~ /(bash|sh|zsh)$/ {print $1, $7}'

sudo apt install -y libpam-pwquality
# /etc/login.defs:
#   PASS_MAX_DAYS 365
#   PASS_MIN_DAYS 1
# apply max-age to an existing user:
sudo chage --maxdays 365 --warndays 14 deploy
sudo chage -l deploy                   # review aging

chmod 600 ~/.ssh/authorized_keys ~/.ssh/id_*    # rw owner only
chmod 700 ~/.ssh                                # rwx owner only
chmod 600 .env config/secrets.yml               # app secrets
sudo chown -R deploy:deploy /srv/myapp

# world-writable files outside /tmp (privilege-escalation risk):
sudo find / -xdev -type f -perm -0002 ! -path '/proc/*' 2>/dev/null
# SUID/SGID binaries — baseline this list; investigate anything unexpected:
sudo find / -xdev -type f \( -perm -4000 -o -perm -2000 \) -exec ls -l {} \; 2>/dev/null

[Unit]
Description=My App
After=network-online.target
Wants=network-online.target

[Service]
# Auto-create a transient unprivileged user; no shell, no home:
DynamicUser=yes
WorkingDirectory=/srv/myapp
ExecStart=/srv/myapp/bin/server
Restart=on-failure
RestartSec=5

# --- Sandboxing (kernel-enforced) ---
NoNewPrivileges=yes          # block setuid escalation
ProtectSystem=strict         # whole FS read-only...
ProtectHome=yes              # ...and /home, /root hidden
ReadWritePaths=/var/lib/myapp   # ...except what it truly needs
PrivateTmp=yes               # private /tmp, /var/tmp
PrivateDevices=yes           # no raw device access
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
RestrictNamespaces=yes
LockPersonality=yes
MemoryDenyWriteExecute=yes
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable --now myapp
systemd-analyze security myapp        # lower "exposure" score = better; aim < 5.0

systemctl list-units --type=service --state=running   # what's running
sudo ss -tulpn                                        # what's LISTENING + pid
systemctl list-unit-files --state=enabled             # what starts at boot
# turn one off completely:
sudo systemctl disable --now cups avahi-daemon 2>/dev/null

sudo tee /etc/sysctl.d/99-hardening.conf >/dev/null <<'EOF'
# --- Spoofing / routing ---
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv6.conf.all.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.log_martians = 1

# --- ICMP / SYN ---
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.tcp_syncookies = 1

# --- Kernel info leaks / exploit mitigation ---
kernel.kptr_restrict = 2
kernel.dmesg_restrict = 1
kernel.randomize_va_space = 2
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
fs.suid_dumpable = 0
EOF
sudo sysctl --system        # load all sysctl.d files now

sudo apt install -y auditd audispd-plugins
sudo systemctl enable --now auditd
# watch a sensitive file for any change:
sudo auditctl -w /etc/passwd -p wa -k passwd_changes
sudo ausearch -k passwd_changes -i        # human-readable matches

sudo mkdir -p /var/log/journal
sudo sed -i 's/^#\?Storage=.*/Storage=persistent/;s/^#\?SystemMaxUse=.*/SystemMaxUse=500M/' /etc/systemd/journald.conf
sudo systemctl restart systemd-journald
journalctl --disk-usage

journalctl -p err -b               # all errors this boot
journalctl -u ssh -g "Accepted|Failed" --since "24 hours ago"
sudo lastb | head                  # recent failed logins (btmp)
last -a | head                     # recent successful logins
who -a                             # who is on right now

# example with restic — verify integrity, then do a trial restore:
restic snapshots
restic check --read-data-subset=10%
restic restore latest --target /tmp/restore-test --include /etc