(simatec) Beta v0.9.0

.gitignore

@@ -0,0 +1,5 @@
+.git
+test.sh
+.vscode
+.DS_Store
+/build/test_build.conf

README.md

@@ -144,6 +144,8 @@ To leave the container on the console you have to execute 2x `exit`.
 **********************************************************************************************************
 
 ## Changelog
+### 0.9.0 (19.01.2023)
+* (simatec) Betatest Piler Build
 
 ### 0.8.3 (19.01.2023)
 * (simatec) Update himself added

build/DOCKERFILE

@@ -0,0 +1,48 @@
+FROM ubuntu:22.04
+
+ARG PACKAGE
+
+LABEL description="piler ubuntu jammy image" \
+      maintainer="Janos SUTO, sj@acts.hu" \
+      package="${PACKAGE}"
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    DISTRO="jammy" \
+    PILER_USER="piler" \
+    MYSQL_DATABASE="piler"
+
+COPY ${PACKAGE} /
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+       wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
+    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
+    dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
+VOLUME ["/var/piler/imap"]
+VOLUME ["/var/piler/stat"]
+VOLUME ["/var/spool/cron/crontabs"]
+
+
+EXPOSE 25 80 443
+
+COPY start.sh /start.sh
+
+CMD ["/start.sh"]

build/build.conf

@@ -0,0 +1,11 @@
+# Piler Package
+
+PILER_PACKAGE="piler_1.4.3-jammy-ee8912fe_amd64.deb"
+
+# Piler Version
+
+PILER_VERSION="1.4.3"
+
+# MariaDB Version
+
+MARIADB_VERSION="10.6"

build/build.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Path Settings
+buildPth=`pwd`
+
+cd $buildPth
+
+# chmod start.sh
+chmod a+x start.sh
+
+# build config load
+. ./build.conf
+
+# set Piler Version
+sed -i 's/PILER_VERSION=.*/PILER_VERSION="'$PILER_VERSION'"/g' ../piler.conf
+
+# set Maria-DB Version
+sed -i 's/MARIADB_VERSION=.*/MARIADB_VERSION="'$MARIADB_VERSION'"/g' ../piler.conf
+MARIADB_VERSION
+
+# Package Download
+rm -f $buildPth/*.deb
+
+curl -o $buildPth/$PILER_PACKAGE https://bitbucket.org/jsuto/piler/downloads/$PILER_PACKAGE
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="simatec/piler:$PILER_VERSION"
+
+if [ ! -f $buildPth/$PILER_PACKAGE ]; then 
+    echo "ERROR: missing package name" 1>&2; exit 1; 
+fi
+
+docker build --build-arg PACKAGE="$PILER_PACKAGE" -t "$IMAGE_NAME" .

build/start.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+
+
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
+}
+
+
+log() {
+   echo "DEBUG:" "$*"
+}
+
+
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
+}
+
+
+fix_configs() {
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
+   fi
+
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
+   fi
+
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
+}
+
+
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   rsyslogd
+   service cron start
+   service php8.1-fpm start
+   service nginx start
+}
+
+
+start_piler() {
+   if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" piler
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
+fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
+
+sleep infinity

config/piler-default.yml

@@ -2,7 +2,7 @@ version: "3"
 services:
 
   mysql:
-    image: mariadb:10.6
+    image: mariadb:${MARIADB_VERSION}
     container_name: mysql_piler
     restart: unless-stopped
     cap_drop:
@@ -36,7 +36,7 @@ services:
     command: -m 64
 
   piler:
-    image: sutoj/piler:1.4.2
+    image: simatec/piler:${PILER_VERSION}
     container_name: piler
     restart: unless-stopped
     hostname: ${PILER_DOMAIN}

config/piler-ssl.yml

@@ -23,7 +23,7 @@ services:
     environment:
       - DEFAULT_EMAIL=${LETSENCRYPT_EMAIL}
   mysql:
-    image: mariadb:10.6
+    image: mariadb:${MARIADB_VERSION}
     container_name: mysql-piler
     restart: unless-stopped
     cap_drop:
@@ -57,7 +57,7 @@ services:
     command: -m 64
 
   piler:
-    image: sutoj/piler:1.4.2
+    image: simatec/piler:${PILER_VERSION}
     container_name: piler
     restart: unless-stopped
     expose:

install-piler.sh

@@ -100,6 +100,7 @@ fi
 installPth=`pwd`
 configPth="$installPth/config"
 etcPth="/var/lib/docker/volumes/piler-docker_piler_etc/_data"
+buildPth="$installPth/build"
 
 ############################## Installer Settings ######################################
 
@@ -274,6 +275,9 @@ if [ ! -f $installPth/.env ]; then
     ln -s ./piler.conf .env
 fi
 
+# Build Piler
+bash $buildPth/build.sh
+
 if [ -f $installPth/docker-compose.yml ]; then
     rm $installPth/docker-compose.yml
 fi

piler.conf.example

@@ -62,3 +62,13 @@ MAILCOW_APIKEY="your_Mailcow_API-Key"
 # your Mailcow Host Domain
 
 MAILCOW_HOST="mail.example.com"
+
+# ------ Container Versions --------
+
+# Piler Version
+
+PILER_VERSION="1.4.3"
+
+# MariaDB Version
+
+MARIADB_VERSION="10.6"

update.sh

@@ -108,6 +108,7 @@ done
 installPth=`pwd`
 configPth="$installPth/config"
 etcPth="/var/lib/docker/volumes/piler-docker_piler_etc/_data"
+buildPth="$installPth/build"
 
 # config load
 . ./piler.conf
@@ -146,6 +147,15 @@ for fileUpdate in install-piler.sh LICENSE piler.conf.example; do
   echo
 done
 
+for buildUpdate in start.sh build.sh DOCKERFILE build.conf; do
+  echo
+  echo "${purple}${HLINE}${HLINE_SMALL}"
+  echo "${purple}****** Download Update $buildUpdate ******"
+  curl -o $buildPth/$buildUpdate https://raw.githubusercontent.com/simatec/piler-docker/main/build/$buildUpdate
+  echo "${purple}${HLINE}${HLINE_SMALL}${normal}"
+  echo
+done
+
 # old docker stop
 cd $installPth
 
@@ -155,6 +165,27 @@ else
   docker-compose down
 fi
 
+# Backup Config
+if [ ! -d $installPth/backup ]; then
+  mkdir -p $installPth/backup
+fi
+
+cp $etcPth/config-site.php.bak $installPth/backup/ && cp $etcPth/config-site.php $installPth/backup/
+cp $etcPth/piler.conf.bak $installPth/backup/ && cp $etcPth/piler.conf $installPth/backup/
+cp $etcPth/.my.cnf $installPth/backup/
+
+# delete old files
+rm -f $etcPth/piler.key $etcPth/piler.pem $etcPth/config-site.php
+
+# Added Manticore
+if [ ! -f $etcPth/MANTICORE ]; then
+  touch $etcPth/MANTICORE
+fi
+
+# Build Piler
+bash $buildPth/build.sh
+
+# Copy docker-compose.yml
 if [ "$USE_LETSENCRYPT" = "yes" ]; then
   cp $configPth/piler-ssl.yml $installPth/docker-compose.yml
 else
@@ -192,79 +223,10 @@ BLA::start_loading_animation "${BLA_metro[@]}"
 sleep 20
 BLA::stop_loading_animation
 
-if [ ! -f $etcPth/config-site.php.bak ]; then
+# Restore Config
-  cp $etcPth/config-site.php $etcPth/config-site.php.bak
+cp $installPth/backup/config-site.php.bak $etcPth/ && cp $installPth/backup/config-site.php $etcPth/
-else
+cp $installPth/backup/piler.conf.bak $etcPth/ && cp $etcPth/piler.conf $etcPth/
-  rm $etcPth/config-site.php
+cp $installPth/backup/.my.cnf $etcPth/
-  cp $etcPth/config-site.php.bak $etcPth/config-site.php
-fi
-
-echo
-echo "${blue}${HLINE}"
-echo "${blue}                       set User settings ..."
-echo "${blue}${HLINE}${normal}"
-echo
-
-cat >> $etcPth/config-site.php <<EOF
-
-// Smarthost
-\$config['SMARTHOST'] = '$SMARTHOST';
-\$config['SMARTHOST_PORT'] = '25';
-
-// CUSTOM
-\$config['PROVIDED_BY'] = '$PILER_DOMAIN';
-\$config['SUPPORT_LINK'] = 'mailto:$SUPPORT_MAIL';
-\$config['COMPATIBILITY'] = '';
-
-// fancy features.
-\$config['ENABLE_INSTANT_SEARCH'] = 1;
-\$config['ENABLE_TABLE_RESIZE'] = 1;
-
-\$config['ENABLE_DELETE'] = 1;
-\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
-
-// general settings.
-\$config['TIMEZONE'] = '$TIME_ZONE';
-
-// authentication
-// Enable authentication against an imap server
-\$config['ENABLE_IMAP_AUTH'] = 1;
-\$config['RESTORE_OVER_IMAP'] = 1;
-\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
-\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-\$config['IMAP_HOST'] = '$IMAP_SERVER';
-\$config['IMAP_PORT'] =  993;
-\$config['IMAP_SSL'] = true;
-
-// authentication against an ldap directory (disabled by default)
-//\$config['ENABLE_LDAP_AUTH'] = 1;
-//\$config['LDAP_HOST'] = '$SMARTHOST';
-//\$config['LDAP_PORT'] = 389;
-//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
-//\$config['LDAP_MAIL_ATTR'] = 'mail';
-//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
-//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
-//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
-
-// authentication against an Uninvention based ldap directory 
-//\$config['ENABLE_LDAP_AUTH'] = 1;
-//\$config['LDAP_HOST'] = '$SMARTHOST';
-//\$config['LDAP_PORT'] = 7389;
-//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
-//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
-//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
-//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
-//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
-//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
-//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
-
-// special settings.
-//\$config['MEMCACHED_ENABLED'] = 1;
-\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
-EOF
 
 if [ "$USE_MAILCOW" = true ]; then
 
@@ -285,23 +247,7 @@ include('auth-mailcow.php');
 EOF
 
 curl -o $etcPth/auth-mailcow.php https://raw.githubusercontent.com/patschi/mailpiler-mailcow-integration/master/auth-mailcow.php
-fi
+fi‚
-
-# add config settings
-
-if [ ! -f $etcPth/piler.conf.bak ]; then
-  cp $etcPth/piler.conf $etcPth/piler.conf.bak
-else
-  rm $etcPth/piler.conf
-  cp $etcPth/piler.conf.bak $etcPth/piler.conf
-fi
-
-sed -i "s/default_retention_days=.*/default_retention_days=$DEFAULT_RETENTION_DAYS/" $etcPth/piler.conf
-sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" $etcPth/piler.conf
-
-cat >> $etcPth/piler.conf <<EOF
-queuedir=/var/piler/store
-EOF
 
 # piler restart
 echo