Merge pull request #157 from haiwen/multi-containers

Multi containers

README.md

@@ -1,6 +1,6 @@
 [![Build Status](https://secure.travis-ci.org/haiwen/seafile-docker.png?branch=master)](http://travis-ci.org/haiwen/seafile-docker)
 
-### About
+## About
 
 - [Docker](https://docker.com/) is an open source project to pack, ship and run any Linux application in a lighter weight, faster container than a traditional virtual machine.
 
@@ -10,6 +10,16 @@
 
 If you are not familiar with docker commands, please refer to [docker documentation](https://docs.docker.com/engine/reference/commandline/cli/).
 
+## For seafile 7.x.x
+
+Starting with 7.0, we have adjusted seafile-docker image to use multiple containers. The old image runs MariaDB-Server and Memcached in the same container with Seafile server. Now, we strip the MariaDB-Server and Memcached services from the Seafile image and run them in their respective containers.
+
+If you plan to deploy seafile 7.0, you should refer to the [Deploy Documentation](https://download.seafile.com/published/support/docker/deploy%20seafile%20with%20docker.md).
+
+If you plan to upgrade 6.3 to 7.0, you can refer to the [Upgrade Documentation](https://download.seafile.com/published/support/docker/6.3%20upgrade%20to%207.0.md).
+
+## For seafile 6.x.x
+
 ### Getting Started
 
 To run the seafile server container:

cluster/image/base/Dockerfile

@@ -7,7 +7,7 @@ ENV UPDATED_AT=20180412 \
 
 CMD ["/sbin/my_init", "--", "bash", "-l"]
 
-RUN apt-get update -qq && apt-get -qq -y install memcached nginx tzdata
+RUN apt-get update -qq && apt-get -qq -y install nginx tzdata
 
 # Utility tools
 RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl

cluster/image/base/my_init.d/99_mysql_setup.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-
-# Init mysql data dir.
-# Borrowed from https://github.com/fideloper/docker-mysql/blob/master/etc/my_init.d/99_mysql_setup.sh
-
-if [[ ! -d /var/lib/mysql/mysql ]]; then
-    echo 'Rebuilding mysql data dir'
-
-    chown -R mysql.mysql /var/lib/mysql
-
-    mysql_install_db >/var/log/mysql-bootstrap.log 2>&1
-    # TODO: print the log if mysql_install_db fails
-
-    rm -rf /var/run/mysqld/*
-
-    echo 'Starting mysqld'
-    mysqld_safe >>/var/log/mysql-bootstrap.log 2>&1 &
-
-    echo 'Waiting for mysqld to come online'
-    # The sleep 1 is there to make sure that inotifywait starts up before the socket is created
-    while [[ ! -S /var/run/mysqld/mysqld.sock ]]; do
-        sleep 1
-    done
-
-    echo 'Fixing root password'
-    /usr/bin/mysqladmin -u root password ''
-
-    # if [ -d /var/lib/mysql/setup ]; then
-    #     echo 'Found /var/lib/mysql/setup - scanning for SQL scripts'
-    #     for sql in $(ls /var/lib/mysql/setup/*.sql 2>/dev/null | sort); do
-    #         echo 'Running script:' $sql
-    #         mysql -uroot -proot -e "\. $sql"
-    #         mv $sql $sql.processed
-    #     done
-    # else
-    #     echo 'No setup directory with extra sql scripts to run'
-    # fi
-
-    echo 'Shutting down mysqld'
-    mysqladmin -uroot  shutdown
-
-    retry=0 maxretry=10
-    while [[ -e /var/run/mysqld/mysqld.sock && $retry -le $maxretry ]]; do
-        retry=$((retry+1))
-        sleep 1
-    done
-fi

cluster/image/base/services/memcached.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-# `/sbin/setuser memcache` runs the given command as the user `memcache`.
-# If you omit that part, the command will be run as root.
-exec /sbin/setuser memcache /usr/bin/memcached >>/var/log/memcached.log 2>&1

cluster/image/base/services/mysql.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e
-
-shutdown_mysql() {
-    if [[ -S /var/run/mysqld/mysqld.sock ]]; then
-        mysqladmin -u root shutdown || true
-    fi
-}
-
-trap shutdown_mysql EXIT
-
-mkdir -p /var/run/mysqld
-chown mysql:mysql /var/run/mysqld
-
-rm -f /var/lib/mysql/aria_log_control
-
-/sbin/setuser mysql /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib/mysql/plugin --user=mysql --skip-log-error --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/run/mysqld/mysqld.sock --port=3306 >/var/log/mysql.log 2>&1

image/Makefile

@@ -1,15 +1,15 @@
-server_version=6.3.13
+server_version=7.0.1
 
-base_image=seafileltd/base:18.04
+base_image=seafileltd/base-mc:18.04
-base_image_squashed=seafileltd/base:18.04-squashed
+base_image_squashed=seafileltd/base-mc:18.04-squashed
-pro_base_image=seafileltd/pro-base:18.04
+pro_base_image=seafileltd/pro-base-mc:18.04
-pro_base_image_squashed=seafileltd/pro-base:18.04-squashed
+pro_base_image_squashed=seafileltd/pro-base-mc:18.04-squashed
-server_image=seafileltd/seafile:$(server_version)
+server_image=seafileltd/seafile-mc:$(server_version)
-server_image_squashed=seafileltd/seafile:$(server_version)-squashed
+server_image_squashed=seafileltd/seafile-mc:$(server_version)-squashed
-pro_server_image=seafileltd/seafile-pro:$(server_version)
+pro_server_image=seafileltd/seafile-pro-mc:$(server_version)
-pro_server_image_squashed=seafileltd/seafile-pro:$(server_version)-squashed
+pro_server_image_squashed=seafileltd/seafile-pro-mc:$(server_version)-squashed
-latest_pro_server_image=seafileltd/seafile-pro:latest
+latest_pro_server_image=seafileltd/seafile-pro-mc:latest
-latest_server_image=seafileltd/seafile:latest
+latest_server_image=seafileltd/seafile-mc:latest
 
 all:
 	@echo

image/base/Dockerfile

@@ -7,7 +7,7 @@ ENV UPDATED_AT=20180412 \
 
 CMD ["/sbin/my_init", "--", "bash", "-l"]
 
-RUN apt-get update -qq && apt-get -qq -y install memcached mariadb-server nginx
+RUN apt-get update -qq && apt-get -qq -y install nginx
 
 # Utility tools
 RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl
@@ -26,22 +26,13 @@ RUN pip install -r /tmp/requirements.txt
 
 COPY services /services
 
-RUN mkdir -p /etc/service/memcached && \
-    mv /services/memcached.sh /etc/service/memcached/run
-
 RUN mkdir -p /etc/service/nginx && \
     rm -f /etc/nginx/sites-enabled/* /etc/nginx/conf.d/* && \
     mv /services/nginx.conf /etc/nginx/nginx.conf && \
     mv /services/nginx.sh /etc/service/nginx/run
 
-RUN mkdir -p /etc/service/mysql && \
-    mv /services/mysql.sh /etc/service/mysql/run
-
 RUN mkdir -p /etc/my_init.d && rm -f /etc/my_init.d/00_regen_ssh_host_keys.sh
-ADD my_init.d/99_mysql_setup.sh /etc/my_init.d/
 
-# Clean up for docker squash
-# See https://github.com/goldmann/docker-squash
 RUN rm -rf \
     /root/.cache \
     /root/.npm \

image/pro_base/Dockerfile

@@ -1,4 +1,4 @@
-FROM seafileltd/base:18.04
+FROM seafileltd/base-mc:18.04
 
 # syslog-ng and syslog-forwarder would mess up the container stdout, not good
 # when debugging/upgrading.
@@ -7,19 +7,17 @@ FROM seafileltd/base:18.04
 # when RUN apt-get
 RUN mkdir -p /usr/share/man/man1
 
-RUN apt update
+RUN apt-get update \
+    && apt-get install -y libmemcached-dev zlib1g-dev pwgen curl openssl poppler-utils libpython2.7 libreoffice \
+    libreoffice-script-provider-python ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy python-requests tzdata \
+    python-pip python-setuptools python-urllib3 python-ldap python-ceph
 
-RUN apt-get install -y openjdk-8-jre libmemcached-dev zlib1g-dev pwgen curl openssl poppler-utils libpython2.7 libreoffice \
+# The S3 storage, oss storage and psd online preview etc,
-libreoffice-script-provider-python ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy python-requests
-
-RUN apt-get install -y tzdata python-pip python-setuptools python-urllib3 python-ldap python-ceph
-
-# The S3 storage, oss storage and psd online preview etc, 
 # depends on the python-backages as follow:
-RUN pip install boto==2.43.0 \ 
+RUN pip install boto==2.43.0 \
-    oss2==2.3.0 \ 
+    oss2==2.3.0 \
-    psd-tools==1.4 \ 
+    psd-tools==1.4 \
-    pycryptodome==3.7.2 \ 
+    pycryptodome==3.7.2 \
     twilio==5.7.0
 
-RUN apt clean
+RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

image/pro_seafile/Dockerfile

@@ -1,7 +1,7 @@
-FROM seafileltd/pro-base:18.04
+FROM seafileltd/pro-base-mc:18.04
 WORKDIR /opt/seafile
 
-ENV SEAFILE_VERSION=6.3.13 SEAFILE_SERVER=seafile-pro-server
+ENV SEAFILE_VERSION=7.0.1 SEAFILE_SERVER=seafile-pro-server
 
 RUN mkdir -p /etc/my_init.d
 

image/seafile/Dockerfile

@@ -1,4 +1,4 @@
-FROM seafileltd/base:18.04
+FROM seafileltd/base-mc:18.04
 WORKDIR /opt/seafile
 
 RUN mkdir -p /etc/my_init.d

scripts/auto_renew_crt.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+
+ssldir=${1:?"error params"}
+domain=${2:?"error params"}
+
+letsencryptdir=$ssldir/letsencrypt
+letsencrypt_script=$letsencryptdir/acme_tiny.py
+
+ssl_account_key=${domain}.account.key
+ssl_csr=${domain}.csr
+ssl_key=${domain}.key
+ssl_crt=${domain}.crt
+renew_cert_script=/scripts/renew_cert.sh
+
+if [[ ! -x ${renew_cert_script} ]]; then
+    cat > ${renew_cert_script} << EOF
+#!/bin/bash
+python ${letsencrypt_script} --account-key ${ssldir}/${ssl_account_key} --csr ${ssldir}/${ssl_csr} --acme-dir /var/www/challenges/ > ${ssldir}/${ssl_crt} || exit
+$(which nginx) -s reload
+EOF
+
+    chmod u+x ${renew_cert_script}
+
+    if [[ ! -d "/var/www/challenges" ]]; then
+        mkdir -p /var/www/challenges
+    fi
+
+    cat >> /etc/crontab << EOF
+00 1    1 * *   root    /scripts/renew_cert.sh 2>> /var/log/acme_tiny.log
+EOF
+
+    echo 'Created a crontab to auto renew the cert for letsencrypt.'
+else
+    echo 'Found existing the script for renew the cert.'
+    echo 'Skip create the crontab for letscncrypt since maybe we have created before.'
+fi

scripts/bootstrap.py

@@ -50,6 +50,9 @@ def init_letsencrypt():
         loginfo('Found existing cert file {}'.format(ssl_crt))
         if cert_has_valid_days(ssl_crt, 30):
             loginfo('Skip letsencrypt verification since we have a valid certificate')
+	    if exists(join(ssl_dir, 'letsencrypt')):
+                # Create a crontab to auto renew the cert for letsencrypt.
+                call('/scripts/auto_renew_crt.sh {0} {1}'.format(ssl_dir, domain))
             return
 
     loginfo('Starting letsencrypt verification')
@@ -70,6 +73,9 @@ def init_letsencrypt():
     #     time.sleep(1000)
     #     sys.exit(1)
 
+    call('/scripts/auto_renew_crt.sh {0} {1}'.format(ssl_dir, domain))
+    # Create a crontab to auto renew the cert for letsencrypt.
+
 
 def generate_local_nginx_conf():
     # Now create the final nginx configuratin
@@ -113,14 +119,22 @@ def init_seafile_server():
         'SERVER_IP': get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com'),
         'MYSQL_USER': 'seafile',
         'MYSQL_USER_PASSWD': str(uuid.uuid4()),
-        'MYSQL_USER_HOST': '127.0.0.1',
+        'MYSQL_USER_HOST': '%.%.%.%',
+	'MYSQL_HOST': get_conf('DB_HOST','127.0.0.1'),
         # Default MariaDB root user has empty password and can only connect from localhost.
-        'MYSQL_ROOT_PASSWD': '',
+        'MYSQL_ROOT_PASSWD': get_conf('DB_ROOT_PASSWD', ''),
     }
 
     # Change the script to allow mysql root password to be empty
-    call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}'''
+    # call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}'''
-         .format(get_script('setup-seafile-mysql.py')))
+    #     .format(get_script('setup-seafile-mysql.py')))
+
+    # Change the script to disable check MYSQL_USER_HOST
+    call('''sed -i -e '/def validate_mysql_user_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}'''
+        .format(get_script('setup-seafile-mysql.py')))
+
+    call('''sed -i -e '/def validate_mysql_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}'''
+        .format(get_script('setup-seafile-mysql.py')))
 
     setup_script = get_script('setup-seafile-mysql.sh')
     call('{} auto -n seafile'.format(setup_script), env=env)
@@ -132,13 +146,15 @@ def init_seafile_server():
         fp.write("""CACHES = {
     'default': {
         'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
-        'LOCATION': '127.0.0.1:11211',
+        'LOCATION': 'memcached:11211',
     },
     'locmem': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
     },
 }
 COMPRESS_CACHE_BACKEND = 'locmem'""")
+        fp.write('\n')
+        fp.write("TIME_ZONE = '{time_zone}'".format(time_zone=os.getenv('TIME_ZONE',default='Etc/UTC')))
         fp.write('\n')
         fp.write('FILE_SERVER_ROOT = "{proto}://{domain}/seafhttp"'.format(proto=proto, domain=domain))
         fp.write('\n')
@@ -155,6 +171,20 @@ COMPRESS_CACHE_BACKEND = 'locmem'""")
         fp.write('UNIX_SOCKET = /opt/seafile/ccnet.sock\n')
         fp.write('\n')
 
+    # Disabled the Elasticsearch process on Seafile-container
+    # Connection to the Elasticsearch-container
+    if os.path.exists(join(topdir, 'conf', 'seafevents.conf')):
+        with open(join(topdir, 'conf', 'seafevents.conf'), 'r') as fp:
+            fp_lines = fp.readlines()
+            if '[INDEX FILES]\n' in fp_lines:
+               insert_index = fp_lines.index('[INDEX FILES]\n') + 1
+               insert_lines = ['es_port = 9200\n', 'es_host = elasticsearch\n', 'external_es_server = true\n']
+               for line in insert_lines:
+                   fp_lines.insert(insert_index, line)
+    
+        with open(join(topdir, 'conf', 'seafevents.conf'), 'w') as fp:
+            fp.writelines(fp_lines)
+
     # After the setup script creates all the files inside the
     # container, we need to move them to the shared volume
     #

scripts/create_data_links.sh

@@ -39,12 +39,6 @@ if [[ ! -e /shared/logs/seafile ]]; then
 fi
 rm -rf /opt/seafile/logs && ln -sf /shared/logs/seafile/ /opt/seafile/logs
 
-rm -rf /var/lib/mysql
-if [[ ! -e /shared/db ]];then
-    mkdir -p /shared/db
-fi
-ln -sf /shared/db /var/lib/mysql
-
 if [[ ! -e /shared/logs/var-log ]]; then
     mv /var/log /shared/logs/var-log
 fi

scripts/utils/__init__.py

@@ -15,6 +15,7 @@ import logging.config
 import click
 import termcolor
 import colorlog
+import MySQLdb
 
 logger = logging.getLogger('.utils')
 
@@ -265,18 +266,19 @@ def update_version_stamp(version, fn=get_version_stamp_file()):
         fp.write(version + '\n')
 
 def wait_for_mysql():
-    tried = False
+    db_host = get_conf('DB_HOST', '127.0.0.1')
-    while not exists('/var/run/mysqld/mysqld.sock'):
+    db_user = 'root'
-        logdbg('waiting for mysql server to be ready')
+    db_passwd = get_conf('DB_ROOT_PASSWD', '')
-        """
+
-        Try to solve the MySQL startup failure caused by the permission problem 
+    while True:
-        caused by the MySQL user's uid change.
+        try:
-        """
+	    MySQLdb.connect(host=db_host, port=3306, user=db_user, passwd=db_passwd)
-        if not tried:
+	except Exception as e:
-            os.system('rm /var/lib/mysql/tc.log -f  && chown -R mysql.mysql /var/lib/mysql/')
+	    print ('waiting for mysql server to be ready: %s', e)
-            tried = True
+	    time.sleep(2)
-        time.sleep(2)
+	    continue
-    logdbg('mysql server is ready')
+	logdbg('mysql server is ready')
+	return
 
 def wait_for_nginx():
     while True: