diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a5115b --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.git +test.sh +.vscode +.DS_Store +/build/test_build.conf \ No newline at end of file diff --git a/README.md b/README.md index e049ec6..7412680 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,8 @@ To leave the container on the console you have to execute 2x `exit`. ********************************************************************************************************** ## Changelog +### 0.9.0 (19.01.2023) +* (simatec) Betatest Piler Build ### 0.8.3 (19.01.2023) * (simatec) Update himself added diff --git a/build/DOCKERFILE b/build/DOCKERFILE new file mode 100644 index 0000000..e2d7abd --- /dev/null +++ b/build/DOCKERFILE @@ -0,0 +1,48 @@ +FROM ubuntu:22.04 + +ARG PACKAGE + +LABEL description="piler ubuntu jammy image" \ + maintainer="Janos SUTO, sj@acts.hu" \ + package="${PACKAGE}" + +ENV DEBIAN_FRONTEND="noninteractive" \ + DISTRO="jammy" \ + PILER_USER="piler" \ + MYSQL_DATABASE="piler" + +COPY ${PACKAGE} / + +RUN apt-get update && \ + apt-get -y --no-install-recommends install \ + wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \ + php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \ + libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \ + wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \ + dpkg -i manticore-repo.noarch.deb && \ + rm -f manticore-repo.noarch.deb && \ + apt-get update && \ + apt-get install -y manticore manticore-columnar-lib && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \ + sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \ + dpkg -i ${PACKAGE} && \ + touch /etc/piler/MANTICORE && \ + ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \ + rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \ + crontab -u $PILER_USER /usr/share/piler/piler.cron + +VOLUME ["/etc/piler"] +VOLUME ["/var/piler/store"] +VOLUME ["/var/piler/manticore"] +VOLUME ["/var/piler/imap"] +VOLUME ["/var/piler/stat"] +VOLUME ["/var/spool/cron/crontabs"] + + +EXPOSE 25 80 443 + +COPY start.sh /start.sh + +CMD ["/start.sh"] \ No newline at end of file diff --git a/build/build.conf b/build/build.conf new file mode 100644 index 0000000..152f6e4 --- /dev/null +++ b/build/build.conf @@ -0,0 +1,11 @@ +# Piler Package + +PILER_PACKAGE="piler_1.4.3-jammy-ee8912fe_amd64.deb" + +# Piler Version + +PILER_VERSION="1.4.3" + +# MariaDB Version + +MARIADB_VERSION="10.6" \ No newline at end of file diff --git a/build/build.sh b/build/build.sh new file mode 100644 index 0000000..02df73f --- /dev/null +++ b/build/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Path Settings +buildPth=`pwd` + +cd $buildPth + +# chmod start.sh +chmod a+x start.sh + +# build config load +. ./build.conf + +# set Piler Version +sed -i 's/PILER_VERSION=.*/PILER_VERSION="'$PILER_VERSION'"/g' ../piler.conf + +# set Maria-DB Version +sed -i 's/MARIADB_VERSION=.*/MARIADB_VERSION="'$MARIADB_VERSION'"/g' ../piler.conf +MARIADB_VERSION + +# Package Download +rm -f $buildPth/*.deb + +curl -o $buildPth/$PILER_PACKAGE https://bitbucket.org/jsuto/piler/downloads/$PILER_PACKAGE + +set -o errexit +set -o pipefail +set -o nounset + +IMAGE_NAME="simatec/piler:$PILER_VERSION" + +if [ ! -f $buildPth/$PILER_PACKAGE ]; then + echo "ERROR: missing package name" 1>&2; exit 1; +fi + +docker build --build-arg PACKAGE="$PILER_PACKAGE" -t "$IMAGE_NAME" . diff --git a/build/start.sh b/build/start.sh new file mode 100644 index 0000000..78ffe18 --- /dev/null +++ b/build/start.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +CONFIG_DIR="/etc/piler" +VOLUME_DIR="/var/piler" +PILER_CONF="${CONFIG_DIR}/piler.conf" +PILER_KEY="${CONFIG_DIR}/piler.key" +PILER_PEM="${CONFIG_DIR}/piler.pem" +PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf" +SPHINX_CONF="${CONFIG_DIR}/manticore.conf" +CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php" +PILER_MY_CNF="${CONFIG_DIR}/.my.cnf" + + +error() { + echo "ERROR:" "$*" 1>&2 + exit 1 +} + + +log() { + echo "DEBUG:" "$*" +} + + +pre_flight_check() { + [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable" + [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable" + [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable" + [[ -v MYSQL_USER ]] || error "Missing MYSQL_USER env variable" + [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable" +} + + +give_it_to_piler() { + local f="$1" + + [[ -f "$f" ]] || error "${f} does not exist, aborting" + + chown "${PILER_USER}:${PILER_USER}" "$f" + chmod 600 "$f" +} + + +make_certificate() { + local f="$1" + local crt="/tmp/1.cert" + local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" + + log "Making an ssl certificate" + + openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null + cat "$crt" >> "$f" + rm -f "$crt" + + give_it_to_piler "$f" +} + + +make_piler_key() { + local f="$1" + + log "Generating piler.key" + + dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null + [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}" + + give_it_to_piler "$f" +} + + +fix_configs() { + [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY" + [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM" + + if [[ ! -f "$PILER_NGINX_CONF" ]]; then + log "Writing ${PILER_NGINX_CONF}" + + cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF" + sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF" + fi + + if [[ ! -f "$PILER_CONF" ]]; then + log "Writing ${PILER_CONF}" + + sed \ + -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \ + -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \ + -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \ + -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \ + -e "s/tls_enable=.*/tls_enable=1/g" \ + -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF" + + { + echo "mysqlhost=${MYSQL_HOSTNAME}" + } >> "$PILER_CONF" + + give_it_to_piler "$PILER_CONF" + fi + + if [[ ! -f "$CONFIG_SITE_PHP" ]]; then + log "Writing ${CONFIG_SITE_PHP}" + + cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP" + + sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP" + + { + echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';" + echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';" + echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';" + echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';" + echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';" + echo "\$config['DB_USERNAME'] = '$MYSQL_USER';" + echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';" + echo "\$config['ENABLE_MEMCACHED'] = 1;" + echo "\$memcached_server = ['memcached', 11211];" + } >> "$CONFIG_SITE_PHP" + fi + + sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \ + -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \ + -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \ + -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \ + -i "$SPHINX_CONF" +} + + +wait_until_mysql_server_is_ready() { + while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done + + log "${MYSQL_HOSTNAME} is ready" +} + + +init_database() { + local table + local has_metadata_table=0 + + wait_until_mysql_server_is_ready + + while read -r table; do + if [[ "$table" == metadata ]]; then has_metadata_table=1; fi + done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables') + + if [[ $has_metadata_table -eq 0 ]]; then + log "no metadata table, creating tables" + + mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql + else + log "metadata table exists" + fi + + if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then + mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0" + fi +} + + +create_my_cnf_files() { + printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \ + "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \ + > "$PILER_MY_CNF" + + give_it_to_piler "$PILER_MY_CNF" +} + + +start_services() { + rsyslogd + service cron start + service php8.1-fpm start + service nginx start +} + + +start_piler() { + if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then + log "main1.spp does not exist, creating index files" + su -c "indexer --all --config ${SPHINX_CONF}" piler + fi + + # No pid file should exist for piler + rm -f /var/run/piler/*pid + + /etc/init.d/rc.searchd start + /etc/init.d/rc.piler start +} + + +pre_flight_check +fix_configs +create_my_cnf_files +init_database +start_services +start_piler + +sleep infinity \ No newline at end of file diff --git a/config/piler-default.yml b/config/piler-default.yml index 58a2a71..7b3f5dc 100644 --- a/config/piler-default.yml +++ b/config/piler-default.yml @@ -2,7 +2,7 @@ version: "3" services: mysql: - image: mariadb:10.6 + image: mariadb:${MARIADB_VERSION} container_name: mysql_piler restart: unless-stopped cap_drop: @@ -36,7 +36,7 @@ services: command: -m 64 piler: - image: sutoj/piler:1.4.2 + image: simatec/piler:${PILER_VERSION} container_name: piler restart: unless-stopped hostname: ${PILER_DOMAIN} diff --git a/config/piler-ssl.yml b/config/piler-ssl.yml index 010ab11..1e096f1 100644 --- a/config/piler-ssl.yml +++ b/config/piler-ssl.yml @@ -23,7 +23,7 @@ services: environment: - DEFAULT_EMAIL=${LETSENCRYPT_EMAIL} mysql: - image: mariadb:10.6 + image: mariadb:${MARIADB_VERSION} container_name: mysql-piler restart: unless-stopped cap_drop: @@ -57,7 +57,7 @@ services: command: -m 64 piler: - image: sutoj/piler:1.4.2 + image: simatec/piler:${PILER_VERSION} container_name: piler restart: unless-stopped expose: diff --git a/install-piler.sh b/install-piler.sh index 064c3d6..865dbbc 100644 --- a/install-piler.sh +++ b/install-piler.sh @@ -100,6 +100,7 @@ fi installPth=`pwd` configPth="$installPth/config" etcPth="/var/lib/docker/volumes/piler-docker_piler_etc/_data" +buildPth="$installPth/build" ############################## Installer Settings ###################################### @@ -274,6 +275,9 @@ if [ ! -f $installPth/.env ]; then ln -s ./piler.conf .env fi +# Build Piler +bash $buildPth/build.sh + if [ -f $installPth/docker-compose.yml ]; then rm $installPth/docker-compose.yml fi diff --git a/piler.conf.example b/piler.conf.example index ff36507..7c34dad 100644 --- a/piler.conf.example +++ b/piler.conf.example @@ -62,3 +62,13 @@ MAILCOW_APIKEY="your_Mailcow_API-Key" # your Mailcow Host Domain MAILCOW_HOST="mail.example.com" + +# ------ Container Versions -------- + +# Piler Version + +PILER_VERSION="1.4.3" + +# MariaDB Version + +MARIADB_VERSION="10.6" diff --git a/update.sh b/update.sh index 4f453b6..c7aa7a9 100644 --- a/update.sh +++ b/update.sh @@ -108,6 +108,7 @@ done installPth=`pwd` configPth="$installPth/config" etcPth="/var/lib/docker/volumes/piler-docker_piler_etc/_data" +buildPth="$installPth/build" # config load . ./piler.conf @@ -146,6 +147,15 @@ for fileUpdate in install-piler.sh LICENSE piler.conf.example; do echo done +for buildUpdate in start.sh build.sh DOCKERFILE build.conf; do + echo + echo "${purple}${HLINE}${HLINE_SMALL}" + echo "${purple}****** Download Update $buildUpdate ******" + curl -o $buildPth/$buildUpdate https://raw.githubusercontent.com/simatec/piler-docker/main/build/$buildUpdate + echo "${purple}${HLINE}${HLINE_SMALL}${normal}" + echo +done + # old docker stop cd $installPth @@ -155,6 +165,27 @@ else docker-compose down fi +# Backup Config +if [ ! -d $installPth/backup ]; then + mkdir -p $installPth/backup +fi + +cp $etcPth/config-site.php.bak $installPth/backup/ && cp $etcPth/config-site.php $installPth/backup/ +cp $etcPth/piler.conf.bak $installPth/backup/ && cp $etcPth/piler.conf $installPth/backup/ +cp $etcPth/.my.cnf $installPth/backup/ + +# delete old files +rm -f $etcPth/piler.key $etcPth/piler.pem $etcPth/config-site.php + +# Added Manticore +if [ ! -f $etcPth/MANTICORE ]; then + touch $etcPth/MANTICORE +fi + +# Build Piler +bash $buildPth/build.sh + +# Copy docker-compose.yml if [ "$USE_LETSENCRYPT" = "yes" ]; then cp $configPth/piler-ssl.yml $installPth/docker-compose.yml else @@ -192,79 +223,10 @@ BLA::start_loading_animation "${BLA_metro[@]}" sleep 20 BLA::stop_loading_animation -if [ ! -f $etcPth/config-site.php.bak ]; then - cp $etcPth/config-site.php $etcPth/config-site.php.bak -else - rm $etcPth/config-site.php - cp $etcPth/config-site.php.bak $etcPth/config-site.php -fi - -echo -echo "${blue}${HLINE}" -echo "${blue} set User settings ..." -echo "${blue}${HLINE}${normal}" -echo - -cat >> $etcPth/config-site.php <> $etcPth/piler.conf <