Reset database tables on update (#351)

* Reset database tables on update

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

Dockerfile

@@ -1,7 +1,7 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 LABEL maintainer Ascensio System SIA <support@onlyoffice.com>
 
-ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 DEBIAN_FRONTEND=noninteractive
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 DEBIAN_FRONTEND=noninteractive PG_VERSION=12
 
 ARG ONLYOFFICE_VALUE=onlyoffice
 
@@ -10,17 +10,20 @@ RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d && \
     apt-get -yq install wget apt-transport-https gnupg locales && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 0x8320ca65cb2de8e5 && \
     locale-gen en_US.UTF-8 && \
+    echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections && \
     apt-get -yq install \
         adduser \
         apt-utils \
         bomstrip \
+        certbot \
+        curl \
+        gconf-service \
         htop \
         libasound2 \
         libboost-regex-dev \
         libcairo2 \
-        libcurl3 \
         libcurl3-gnutls \
-        libgconf2-4 \
+        libcurl4 \
         libgtk-3-0 \
         libnspr4 \
         libnss3 \
@@ -31,7 +34,7 @@ RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d && \
         mysql-client \
         nano \
         net-tools \
-        netcat \
+        netcat-openbsd \
         nginx-extras \
         postgresql \
         postgresql-client \
@@ -41,12 +44,15 @@ RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d && \
         software-properties-common \
         sudo \
         supervisor \
+        ttf-mscorefonts-installer \
         xvfb \
         zlib1g && \
+    if [  $(ls -l /usr/share/fonts/truetype/msttcorefonts | wc -l) -ne 61 ]; \
+        then echo 'msttcorefonts failed to download'; exit 1; fi  && \
     echo "SERVER_ADDITIONAL_ERL_ARGS=\"+S 1:1\"" | tee -a /etc/rabbitmq/rabbitmq-env.conf && \
     sed -i "s/bind .*/bind 127.0.0.1/g" /etc/redis/redis.conf && \
     sed 's|\(application\/zip.*\)|\1\n    application\/wasm wasm;|' -i /etc/nginx/mime.types && \
-    pg_conftool 10 main set listen_addresses 'localhost' && \
+    pg_conftool $PG_VERSION main set listen_addresses 'localhost' && \
     service postgresql restart && \
     sudo -u postgres psql -c "CREATE DATABASE $ONLYOFFICE_VALUE;" && \
     sudo -u postgres psql -c "CREATE USER $ONLYOFFICE_VALUE WITH password '$ONLYOFFICE_VALUE';" && \
@@ -63,23 +69,24 @@ COPY run-document-server.sh /app/ds/run-document-server.sh
 
 EXPOSE 80 443
 
-ARG REPO_URL="deb http://download.onlyoffice.com/repo/debian squeeze main"
 ARG COMPANY_NAME=onlyoffice
 ARG PRODUCT_NAME=documentserver
+ARG PACKAGE_URL="http://download.onlyoffice.com/install/documentserver/linux/${COMPANY_NAME}-${PRODUCT_NAME}_amd64.deb"
 
 ENV COMPANY_NAME=$COMPANY_NAME \
     PRODUCT_NAME=$PRODUCT_NAME
 
-RUN echo "$REPO_URL" | tee /etc/apt/sources.list.d/ds.list && \
+RUN wget -q -P /tmp "$PACKAGE_URL" && \
     apt-get -y update && \
     service postgresql start && \
-    apt-get -yq install $COMPANY_NAME-$PRODUCT_NAME && \
+    apt-get -yq install /tmp/$(basename "$PACKAGE_URL") && \
     service postgresql stop && \
     service supervisor stop && \
     chmod 755 /app/ds/*.sh && \
+    rm -f /tmp/$(basename "$PACKAGE_URL") && \
     rm -rf /var/log/$COMPANY_NAME && \
     rm -rf /var/lib/apt/lists/*
 
 VOLUME /var/log/$COMPANY_NAME /var/lib/$COMPANY_NAME /var/www/$COMPANY_NAME/Data /var/lib/postgresql /var/lib/rabbitmq /var/lib/redis /usr/share/fonts/truetype/custom
 
-ENTRYPOINT /app/ds/run-document-server.sh
+ENTRYPOINT ["/app/ds/run-document-server.sh"]

Makefile

@@ -4,14 +4,16 @@ PRODUCT_NAME ?= DocumentServer
 PRODUCT_VERSION ?= 0.0.0
 BUILD_NUMBER ?= 0
 ONLYOFFICE_VALUE ?= onlyoffice
+S3_BUCKET ?= repo-doc-onlyoffice-com
+RELEASE_BRANCH ?= unstable
 
 COMPANY_NAME_LOW = $(shell echo $(COMPANY_NAME) | tr A-Z a-z)
 PRODUCT_NAME_LOW = $(shell echo $(PRODUCT_NAME) | tr A-Z a-z)
 COMPANY_NAME_LOW_ESCAPED = $(subst -,,$(COMPANY_NAME_LOW))
 
+PACKAGE_NAME := $(COMPANY_NAME_LOW)-$(PRODUCT_NAME_LOW)
 PACKAGE_VERSION := $(PRODUCT_VERSION)-$(BUILD_NUMBER)
-
-REPO_URL := "deb [trusted=yes] http://repo-doc-onlyoffice-com.s3.amazonaws.com/ubuntu/trusty/$(COMPANY_NAME_LOW)-$(PRODUCT_NAME_LOW)/$(GIT_BRANCH)/$(PACKAGE_VERSION)/ repo/"
+PACKAGE_URL := http://$(S3_BUCKET).s3.amazonaws.com/$(COMPANY_NAME_LOW)/$(RELEASE_BRANCH)/ubuntu/$(PACKAGE_NAME)_$(PACKAGE_VERSION)_amd64.deb
 
 UPDATE_LATEST := false
 
@@ -35,12 +37,14 @@ DOCKER_TARGETS := $(foreach TAG,$(DOCKER_TAGS),$(DOCKER_REPO)$(COLON)$(TAG))
 
 DOCKER_ARCH := $(COMPANY_NAME_LOW)-$(PRODUCT_NAME_LOW)_$(PACKAGE_VERSION).tar.gz
 
+DOCKER_ARCH_URI := $(COMPANY_NAME_LOW)/$(RELEASE_BRANCH)/docker/$(notdir $(DOCKER_ARCH))
+
 .PHONY: all clean clean-docker deploy docker publish
 
 $(DOCKER_TARGETS): $(DEB_REPO_DATA)
-
+	docker pull ubuntu:20.04
 	docker build \
-		--build-arg REPO_URL=$(REPO_URL) \
+		--build-arg PACKAGE_URL=$(PACKAGE_URL) \
 		--build-arg COMPANY_NAME=$(COMPANY_NAME_LOW) \
 		--build-arg PRODUCT_NAME=$(PRODUCT_NAME_LOW) \
 		--build-arg ONLYOFFICE_VALUE=$(ONLYOFFICE_VALUE) \
@@ -67,7 +71,5 @@ deploy: $(DOCKER_TARGETS)
 		done;)
 
 publish: $(DOCKER_ARCH)
-	aws s3 cp \
-		$(DOCKER_ARCH) \
-		s3://repo-doc-onlyoffice-com.s3.amazonaws.com/docker/amd64/ \
-		--acl public-read
+	aws s3 cp --no-progress --acl public-read \
+		$(DOCKER_ARCH) s3://$(S3_BUCKET)/$(DOCKER_ARCH_URI)

README.md

@@ -103,7 +103,14 @@ So you need to create and install the following files:
         /app/onlyoffice/DocumentServer/data/certs/tls.key
         /app/onlyoffice/DocumentServer/data/certs/tls.crt
 
-When using CA certified certificates, these files are provided to you by the CA. When using self-signed certificates you need to generate these files yourself. Skip the following section if you have CA certified SSL certificates (e.g. [Let's Encrypt](https://letsencrypt.org)).
+When using CA certified certificates (e.g [Let's encrypt](https://letsencrypt.org)), these files are provided to you by the CA. If you are using self-signed certificates you need to generate these files [yourself](#generation-of-self-signed-certificates).
+
+#### Using the automatically generated Let's Encrypt SSL Certificates
+
+        sudo docker run -i -t -d -p 443:443 \
+        -e LETS_ENCRYPT_DOMAIN=your_domain -e LETS_ENCRYPT_MAIL=your_mail  onlyoffice/documentserver
+
+If you want to get and extend Let's Encrypt SSL Certificates automatically just set LETS_ENCRYPT_DOMAIN and LETS_ENCRYPT_MAIL variables.
 
 #### Generation of Self Signed Certificates
 
@@ -185,10 +192,13 @@ Below is the complete list of parameters that can be set using environment varia
 - **JWT_HEADER**: Defines the http header that will be used to send the JSON Web Token. Defaults to `Authorization`.
 - **JWT_IN_BODY**: Specifies the enabling the token validation in the request body to the ONLYOFFICE Document Server. Defaults to `false`.
 - **USE_UNAUTHORIZED_STORAGE**: Set to `true`if using selfsigned certificates for your storage server e.g. Nextcloud. Defaults to `false`
+- **GENERATE_FONTS**: When 'true' regenerates fonts list and the fonts thumbnails etc. at each start. Defaults to `true`
 - **METRICS_ENABLED**: Specifies the enabling StatsD for ONLYOFFICE Document Server. Defaults to `false`.
 - **METRICS_HOST**: Defines StatsD listening host. Defaults to `localhost`.
 - **METRICS_PORT**: Defines StatsD listening port. Defaults to `8125`.
 - **METRICS_PREFIX**: Defines StatsD metrics prefix for backend services. Defaults to `ds.`.
+- **LETS_ENCRYPT_DOMAIN**: Defines the domain for Let's Encrypt certificate.
+- **LETS_ENCRYPT_MAIL**: Defines the domain administator mail address for Let's Encrypt certificate.
 
 ## Installing ONLYOFFICE Document Server integrated with Community and Mail Servers
 

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - '443:443'
     stdin_open: true
     restart: always
+    stop_grace_period: 60s
     volumes:
        - /var/www/onlyoffice/Data
        - /var/log/onlyoffice

run-document-server.sh

@@ -1,20 +1,42 @@
 #!/bin/bash
 
+function clean_exit {
+  /usr/bin/documentserver-prepare4shutdown.sh
+}
+
+trap clean_exit SIGTERM
+
 # Define '**' behavior explicitly
 shopt -s globstar
 
 APP_DIR="/var/www/${COMPANY_NAME}/documentserver"
 DATA_DIR="/var/www/${COMPANY_NAME}/Data"
+PRIVATE_DATA_DIR="${DATA_DIR}/.private"
+DS_RELEASE_DATE="${PRIVATE_DATA_DIR}/ds_release_date"
 LOG_DIR="/var/log/${COMPANY_NAME}"
 DS_LOG_DIR="${LOG_DIR}/documentserver"
 LIB_DIR="/var/lib/${COMPANY_NAME}"
 DS_LIB_DIR="${LIB_DIR}/documentserver"
 CONF_DIR="/etc/${COMPANY_NAME}/documentserver"
+IS_UPGRADE="false"
 
 ONLYOFFICE_DATA_CONTAINER=${ONLYOFFICE_DATA_CONTAINER:-false}
 ONLYOFFICE_DATA_CONTAINER_HOST=${ONLYOFFICE_DATA_CONTAINER_HOST:-localhost}
 ONLYOFFICE_DATA_CONTAINER_PORT=80
 
+RELEASE_DATE="$(stat -c="%y" ${APP_DIR}/server/DocService/docservice | sed -r 's/=([0-9]+)-([0-9]+)-([0-9]+) ([0-9:.+ ]+)/\1-\2-\3/')";
+if [ -f ${DS_RELEASE_DATE} ]; then
+  PREV_RELEASE_DATE=$(head -n 1 ${DS_RELEASE_DATE})
+else
+  PREV_RELEASE_DATE="0"
+fi
+
+if [ "${RELEASE_DATE}" != "${PREV_RELEASE_DATE}" ]; then
+  if [ ${ONLYOFFICE_DATA_CONTAINER} != "true" ]; then
+    IS_UPGRADE="true";
+  fi
+fi
+
 SSL_CERTIFICATES_DIR="${DATA_DIR}/certs"
 if [[ -z $SSL_CERTIFICATE_PATH ]] && [[ -f ${SSL_CERTIFICATES_DIR}/onlyoffice.crt ]]; then
   SSL_CERTIFICATE_PATH=${SSL_CERTIFICATES_DIR}/onlyoffice.crt
@@ -49,6 +71,8 @@ JWT_SECRET=${JWT_SECRET:-secret}
 JWT_HEADER=${JWT_HEADER:-Authorization}
 JWT_IN_BODY=${JWT_IN_BODY:-false}
 
+GENERATE_FONTS=${GENERATE_FONTS:-true}
+
 if [[ ${PRODUCT_NAME} == "documentserver" ]]; then
   REDIS_ENABLED=false
 else
@@ -67,13 +91,18 @@ JSON_EXAMPLE="${JSON_BIN} -q -f ${ONLYOFFICE_EXAMPLE_CONFIG}"
 LOCAL_SERVICES=()
 
 PG_ROOT=/var/lib/postgresql
-PG_VERSION=10
 PG_NAME=main
 PGDATA=${PG_ROOT}/${PG_VERSION}/${PG_NAME}
 PG_NEW_CLUSTER=false
 RABBITMQ_DATA=/var/lib/rabbitmq
 REDIS_DATA=/var/lib/redis
 
+if [ "${LETS_ENCRYPT_DOMAIN}" != "" -a "${LETS_ENCRYPT_MAIL}" != "" ]; then
+  LETSENCRYPT_ROOT_DIR="/etc/letsencrypt/live"
+  SSL_CERTIFICATE_PATH=${LETSENCRYPT_ROOT_DIR}/${LETS_ENCRYPT_DOMAIN}/fullchain.pem
+  SSL_KEY_PATH=${LETSENCRYPT_ROOT_DIR}/${LETS_ENCRYPT_DOMAIN}/privkey.pem
+fi
+
 read_setting(){
   deprecated_var POSTGRESQL_SERVER_HOST DB_HOST
   deprecated_var POSTGRESQL_SERVER_PORT DB_PORT
@@ -90,6 +119,7 @@ read_setting(){
   METRICS_PREFIX="${METRICS_PREFIX:-.ds}"
 
   DB_HOST=${DB_HOST:-${POSTGRESQL_SERVER_HOST:-$(${JSON} services.CoAuthoring.sql.dbHost)}}
+  DB_TYPE=${DB_TYPE:-$(${JSON} services.CoAuthoring.sql.type)}
   case $DB_TYPE in
     "postgres")
       DB_PORT=${DB_PORT:-"5432"}
@@ -108,7 +138,6 @@ read_setting(){
   DB_NAME=${DB_NAME:-${POSTGRESQL_SERVER_DB_NAME:-$(${JSON} services.CoAuthoring.sql.dbName)}}
   DB_USER=${DB_USER:-${POSTGRESQL_SERVER_USER:-$(${JSON} services.CoAuthoring.sql.dbUser)}}
   DB_PWD=${DB_PWD:-${POSTGRESQL_SERVER_PASS:-$(${JSON} services.CoAuthoring.sql.dbPass)}}
-  DB_TYPE=${DB_TYPE:-$(${JSON} services.CoAuthoring.sql.type)}
 
   RABBITMQ_SERVER_URL=${RABBITMQ_SERVER_URL:-$(${JSON} rabbitmq.url)}
   AMQP_URI=${AMQP_URI:-${AMQP_SERVER_URL:-${RABBITMQ_SERVER_URL}}}
@@ -314,21 +343,43 @@ create_db_tbl() {
   esac
 }
 
-create_postgresql_tbl() {
-  CONNECTION_PARAMS="-h$DB_HOST -p$DB_PORT -U$DB_USER -w"
+upgrade_db_tbl() {
+  case $DB_TYPE in
+    "postgres")
+      upgrade_postgresql_tbl
+    ;;
+    "mariadb"|"mysql")
+      upgrade_mysql_tbl
+    ;;
+  esac
+}
+
+upgrade_postgresql_tbl() {
   if [ -n "$DB_PWD" ]; then
     export PGPASSWORD=$DB_PWD
   fi
 
-  PSQL="psql -q $CONNECTION_PARAMS"
-  CREATEDB="createdb $CONNECTION_PARAMS"
+  PSQL="psql -q -h$DB_HOST -p$DB_PORT -d$DB_NAME -U$DB_USER -w"
 
-  # Create db on remote server
-  if $PSQL -lt | cut -d\| -f 1 | grep -qw $DB_NAME | grep 0; then
-    $CREATEDB $DB_NAME
+  $PSQL -f "$APP_DIR/server/schema/postgresql/removetbl.sql"
+  $PSQL -f "$APP_DIR/server/schema/postgresql/createdb.sql"
+}
+
+upgrade_mysql_tbl() {
+  CONNECTION_PARAMS="-h$DB_HOST -P$DB_PORT -u$DB_USER -p$DB_PWD -w"
+  MYSQL="mysql -q $CONNECTION_PARAMS"
+
+  $MYSQL $DB_NAME < "$APP_DIR/server/schema/mysql/removetbl.sql" >/dev/null 2>&1
+  $MYSQL $DB_NAME < "$APP_DIR/server/schema/mysql/createdb.sql" >/dev/null 2>&1
+}
+
+create_postgresql_tbl() {
+  if [ -n "$DB_PWD" ]; then
+    export PGPASSWORD=$DB_PWD
   fi
 
-  $PSQL -d "$DB_NAME" -f "$APP_DIR/server/schema/postgresql/createdb.sql"
+  PSQL="psql -q -h$DB_HOST -p$DB_PORT -d$DB_NAME -U$DB_USER -w"
+  $PSQL -f "$APP_DIR/server/schema/postgresql/createdb.sql"
 }
 
 create_mysql_tbl() {
@@ -418,6 +469,11 @@ update_logrotate_settings(){
   sed 's|\(^su\b\).*|\1 root root|' -i /etc/logrotate.conf
 }
 
+update_release_date(){
+  mkdir -p ${PRIVATE_DATA_DIR}
+  echo ${RELEASE_DATE} > ${DS_RELEASE_DATE}
+}
+
 # create base folders
 for i in converter docservice spellchecker metrics; do
   mkdir -p "${DS_LOG_DIR}/$i"
@@ -522,6 +578,11 @@ if [ ${ONLYOFFICE_DATA_CONTAINER} != "true" ]; then
     waiting_for_redis
   fi
 
+  if [ "${IS_UPGRADE}" = "true" ]; then
+    upgrade_db_tbl
+    update_release_date
+  fi
+
   update_nginx_settings
 
   update_supervisor_settings
@@ -536,8 +597,17 @@ fi
 # it run in all cases.
 service nginx start
 
+if [ "${LETS_ENCRYPT_DOMAIN}" != "" -a "${LETS_ENCRYPT_MAIL}" != "" ]; then
+  if [ ! -f "${SSL_CERTIFICATE_PATH}" -a ! -f "${SSL_KEY_PATH}" ]; then
+    documentserver-letsencrypt.sh ${LETS_ENCRYPT_MAIL} ${LETS_ENCRYPT_DOMAIN}
+  fi
+fi
+
 # Regenerate the fonts list and the fonts thumbnails
-documentserver-generate-allfonts.sh ${ONLYOFFICE_DATA_CONTAINER}
+if [ "${GENERATE_FONTS}" == "true" ]; then
+  documentserver-generate-allfonts.sh ${ONLYOFFICE_DATA_CONTAINER}
+fi
 documentserver-static-gzip.sh ${ONLYOFFICE_DATA_CONTAINER}
 
-tail -f /var/log/${COMPANY_NAME}/**/*.log
+tail -f /var/log/${COMPANY_NAME}/**/*.log &
+wait $!

tests/test.sh

@@ -36,7 +36,7 @@ fi
 # Run test environment
 docker-compose -p ds -f $config up -d
 
-wakeup_timeout=30
+wakeup_timeout=90
 
 # Get documentserver healthcheck status
 echo "Wait for service wake up"