Installation script for Mail Piler (open source version) with Manticore on Ubuntu 24.04 Noble (credit cmer)

piler.sh

#!/bin/bash
#
#
# This script installs the latest open source version of Mail Piler (mailpiler.org) from the master
# branch on GitHub by compiling it from source. It also installs all dependencies, including a MySQL database.
#
# You should run this script as root on a vanilla Ubuntu 24.04 installation.
#
#
set -o errexit
set -o pipefail
set -o nounset
set -x

PILER_HOSTNAME="${PILER_HOSTNAME:-piler.yourdomain.com}"
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-yourpassword}"
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-yourpassword}"
SERVER_ID="${SERVER_ID:-0}"
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
PHP_FPM_SOCKET="/var/run/php/php8.3-fpm.sock"

MYSQL_HOSTNAME="localhost"
MYSQL_DATABASE="piler"
MYSQL_USERNAME="piler"

PILER_TARBALL="https://github.com/jsuto/piler/archive/refs/heads/master.zip"
PILER_USER="piler"
CONFIG_SITE_PHP="/etc/piler/config-site.php"
CONFIG_SITE_DIST_PHP="/etc/piler/config-site.dist.php"

export DEBIAN_FRONTEND=noninteractive

install_prerequisites() {
   apt-get update
   apt-get -y --no-install-recommends install \
      wget rsyslog openssl sysstat php8.3-cli php8.3-cgi php8.3-mysql php8.3-fpm php8.3-zip php8.3-ldap \
      php8.3-gd php8.3-curl php8.3-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip-dev \
      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client \
      mariadb-server build-essential libssl-dev libtre-dev libzip-dev libcurl4-openssl-dev unzip
}

install_manticore() {
	wget https://repo.manticoresearch.com/manticore-repo.noarch.deb
	dpkg -i manticore-repo.noarch.deb
	apt update
	apt -y install manticore manticore-extra
}	


create_user() {
   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --no-create-home --disabled-password --disabled-login --gecos "" $PILER_USER
#   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --disabled-password --gecos "" --shell /bin/bash $PILER_USER
}

create_mysql_user_and_database() {
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER IF NOT EXISTS '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SET PASSWORD FOR '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' = PASSWORD('$MYSQL_PILER_PASSWORD');"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE CHARACTER SET 'utf8mb4';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $MYSQL_DATABASE.* to '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' IDENTIFIED BY '$MYSQL_PILER_PASSWORD';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "FLUSH PRIVILEGES;"

   mysql -u $MYSQL_USERNAME -p"$MYSQL_PILER_PASSWORD" $MYSQL_DATABASE < /etc/piler/db-mysql.sql
}

fix_mysql_settings() {
   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
[mysqld]
innodb_buffer_pool_size=512M
innodb_flush_log_at_trx_commit=1
innodb_log_buffer_size=64M
innodb_log_file_size=64M
innodb_read_io_threads=4
innodb_write_io_threads=4
innodb_log_files_in_group=2
innodb_file_per_table
PILER_CNF
}

start_mysql() {
   fix_mysql_settings
   service mysql restart
}

install_piler() {
   mkdir -p tmp && wget "${PILER_TARBALL}" -O tmp/piler-master.zip
   unzip tmp/piler-master -d tmp
   pushd tmp/piler-master
   ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb --enable-tcpwrappers --enable-memcached
   make clean all install
   cp ./contrib/webserver/piler-nginx.conf /etc/piler/piler-nginx.conf.dist
   cp ./etc/sphinx.conf.dist /etc/piler/sphinx.conf
   cp ./util/db-mysql.sql /etc/piler/db-mysql.sql
   popd

   crontab -u "$PILER_USER" /usr/share/piler/piler.cron

   touch /var/piler/.bash_history
   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
}

create_my_cnf() {
   local user=$1
   local password=$2
   local my_cnf=$3

   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
   chown $PILER_USER:$PILER_USER "$my_cnf"
   chmod 600 "$my_cnf"
}


fix_config_site_php() {
   cp $CONFIG_SITE_DIST_PHP $CONFIG_SITE_PHP
   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"

   {
      echo "\$config['SERVER_ID'] = $SERVER_ID;"
      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
      echo "\$config['SPHINX_VERSION'] = 331;"
   } >> "$CONFIG_SITE_PHP"

   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
   fi

   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
}


add_systemd_services() {
   pushd /etc/systemd/system

   ln -sf /usr/libexec/piler/pilersearch.service .
   ln -sf /usr/libexec/piler/piler.service .
   ln -sf /usr/libexec/piler/piler-smtp.service .

   popd

   systemctl daemon-reload

   systemctl enable pilersearch
   systemctl enable piler
   systemctl enable piler-smtp
}

create_cipher_key() {
   dd if=/dev/urandom bs=56 count=1 of=/etc/piler/piler.key
   chmod 640 /etc/piler/piler.key
   chown piler:piler /etc/piler/piler.key
}


fix_configs() {
	touch /etc/piler/MANTICORE
   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
      ln -s /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf

      nginx -t
      nginx -s reload
   fi

   if [[ ! -f /etc/piler/piler.conf ]]; then
      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
      chmod 600 /etc/piler/piler.conf
      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
   fi

	cp /etc/piler/manticore.conf.dist /etc/piler/manticore.conf
   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
          /etc/piler/manticore.conf
	
}

create_run_piler() {
cat <<EOF > /etc/systemd/system/pilerrun.service
[Unit]
Description=Create piler directory in /var/run
After=network.target

[Service]
Type=simple
ExecStartPre=-/usr/bin/mkdir /var/run/piler
ExecStart=/usr/bin/chown piler:piler /var/run/piler
Restart=on-abort

[Install]
WantedBy=multi-user.target
EOF

systemctl start pilerrun.service
systemctl enable pilerrun.service
}

install_prerequisites
install_manticore
create_user
install_piler
create_mysql_user_and_database
start_mysql

create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf

fix_configs
fix_config_site_php

add_systemd_services
create_cipher_key
create_run_piler

systemctl start pilersearch
systemctl start piler
systemctl start piler-smtp

Nice script, but package libcurl4-openssl-dev required by piler building process but not included in this script.

For those who trying to import mail from imap.yandex.com - you need to change regex in /usr/libexec/piler/imapfetch.py:58
Old str: f = re.split(r' \"[\/\.\\]+\" ', folder)
New str: f = re.split(r'\"\|\"\s*(\S+)', folder)

I Added libcurl4-openssl-dev to the install and checks for file existence before attempting operations so you can continue where left off , and completion message , how do you add the updated code?

I added libcurl4-openssl-dev to the prerequisites. I didn't have an issue on my installs, but can't hurt much to add it if some distribution needs it.

Strange, I restarted the server and stopped receiving emails in Piler. Is there any log I should see? Apologies if I'm off topic when asking this question.

piler -v
1.4.7

root@archive:/etc/piler$ systemctl status piler-smtp

 `piler-smtp.service - Piler SMTP server
 Loaded: loaded (/etc/systemd/system/piler-smtp.service; enabled; preset: enabled)
 Active: active (running) since Mon 2025-06-23 18:23:28 CST; 19min ago`

Strange, I restarted the server and stopped receiving emails in Piler. Is there any log I should see? Apologies if I'm off topic when asking this question.

Manticore 10 broke Piler. See what version of searchd you are running. If 10.1 you might want to backrev Manticore. You can see more details here:
jsuto/piler#292

Strange, I restarted the server and stopped receiving emails in Piler. Is there any log I should see? Apologies if I'm off topic when asking this question.

Manticore 10 broke Piler. See what version of searchd you are running. If 10.1 you might want to backrev Manticore. You can see more details here: jsuto/piler#292

Thanks, I'll try it, I'll let you know.