diff --git a/cluster/image/base/Dockerfile b/cluster/image/base/Dockerfile index d4897de..7d0fd69 100644 --- a/cluster/image/base/Dockerfile +++ b/cluster/image/base/Dockerfile @@ -7,7 +7,7 @@ ENV UPDATED_AT=20180412 \ CMD ["/sbin/my_init", "--", "bash", "-l"] -RUN apt-get update -qq && apt-get -qq -y install memcached nginx tzdata +RUN apt-get update -qq && apt-get -qq -y install nginx tzdata # Utility tools RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl diff --git a/cluster/image/base/my_init.d/99_mysql_setup.sh b/cluster/image/base/my_init.d/99_mysql_setup.sh deleted file mode 100755 index 0db9cde..0000000 --- a/cluster/image/base/my_init.d/99_mysql_setup.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Init mysql data dir. -# Borrowed from https://github.com/fideloper/docker-mysql/blob/master/etc/my_init.d/99_mysql_setup.sh - -if [[ ! -d /var/lib/mysql/mysql ]]; then - echo 'Rebuilding mysql data dir' - - chown -R mysql.mysql /var/lib/mysql - - mysql_install_db >/var/log/mysql-bootstrap.log 2>&1 - # TODO: print the log if mysql_install_db fails - - rm -rf /var/run/mysqld/* - - echo 'Starting mysqld' - mysqld_safe >>/var/log/mysql-bootstrap.log 2>&1 & - - echo 'Waiting for mysqld to come online' - # The sleep 1 is there to make sure that inotifywait starts up before the socket is created - while [[ ! -S /var/run/mysqld/mysqld.sock ]]; do - sleep 1 - done - - echo 'Fixing root password' - /usr/bin/mysqladmin -u root password '' - - # if [ -d /var/lib/mysql/setup ]; then - # echo 'Found /var/lib/mysql/setup - scanning for SQL scripts' - # for sql in $(ls /var/lib/mysql/setup/*.sql 2>/dev/null | sort); do - # echo 'Running script:' $sql - # mysql -uroot -proot -e "\. $sql" - # mv $sql $sql.processed - # done - # else - # echo 'No setup directory with extra sql scripts to run' - # fi - - echo 'Shutting down mysqld' - mysqladmin -uroot shutdown - - retry=0 maxretry=10 - while [[ -e /var/run/mysqld/mysqld.sock && $retry -le $maxretry ]]; do - retry=$((retry+1)) - sleep 1 - done -fi diff --git a/cluster/image/base/services/memcached.sh b/cluster/image/base/services/memcached.sh deleted file mode 100755 index acebfdc..0000000 --- a/cluster/image/base/services/memcached.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# `/sbin/setuser memcache` runs the given command as the user `memcache`. -# If you omit that part, the command will be run as root. -exec /sbin/setuser memcache /usr/bin/memcached >>/var/log/memcached.log 2>&1 diff --git a/cluster/image/base/services/mysql.sh b/cluster/image/base/services/mysql.sh deleted file mode 100755 index 4b24039..0000000 --- a/cluster/image/base/services/mysql.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -shutdown_mysql() { - if [[ -S /var/run/mysqld/mysqld.sock ]]; then - mysqladmin -u root shutdown || true - fi -} - -trap shutdown_mysql EXIT - -mkdir -p /var/run/mysqld -chown mysql:mysql /var/run/mysqld - -rm -f /var/lib/mysql/aria_log_control - -/sbin/setuser mysql /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib/mysql/plugin --user=mysql --skip-log-error --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/run/mysqld/mysqld.sock --port=3306 >/var/log/mysql.log 2>&1 diff --git a/image/base/Dockerfile b/image/base/Dockerfile index e51c4fd..256d418 100644 --- a/image/base/Dockerfile +++ b/image/base/Dockerfile @@ -7,7 +7,7 @@ ENV UPDATED_AT=20180412 \ CMD ["/sbin/my_init", "--", "bash", "-l"] -RUN apt-get update -qq && apt-get -qq -y install memcached mariadb-server nginx +RUN apt-get update -qq && apt-get -qq -y install nginx # Utility tools RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl @@ -26,22 +26,13 @@ RUN pip install -r /tmp/requirements.txt COPY services /services -RUN mkdir -p /etc/service/memcached && \ - mv /services/memcached.sh /etc/service/memcached/run - RUN mkdir -p /etc/service/nginx && \ rm -f /etc/nginx/sites-enabled/* /etc/nginx/conf.d/* && \ mv /services/nginx.conf /etc/nginx/nginx.conf && \ mv /services/nginx.sh /etc/service/nginx/run -RUN mkdir -p /etc/service/mysql && \ - mv /services/mysql.sh /etc/service/mysql/run - RUN mkdir -p /etc/my_init.d && rm -f /etc/my_init.d/00_regen_ssh_host_keys.sh -ADD my_init.d/99_mysql_setup.sh /etc/my_init.d/ -# Clean up for docker squash -# See https://github.com/goldmann/docker-squash RUN rm -rf \ /root/.cache \ /root/.npm \ diff --git a/image/base/my_init.d/99_mysql_setup.sh b/image/base/my_init.d/99_mysql_setup.sh deleted file mode 100755 index 0db9cde..0000000 --- a/image/base/my_init.d/99_mysql_setup.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Init mysql data dir. -# Borrowed from https://github.com/fideloper/docker-mysql/blob/master/etc/my_init.d/99_mysql_setup.sh - -if [[ ! -d /var/lib/mysql/mysql ]]; then - echo 'Rebuilding mysql data dir' - - chown -R mysql.mysql /var/lib/mysql - - mysql_install_db >/var/log/mysql-bootstrap.log 2>&1 - # TODO: print the log if mysql_install_db fails - - rm -rf /var/run/mysqld/* - - echo 'Starting mysqld' - mysqld_safe >>/var/log/mysql-bootstrap.log 2>&1 & - - echo 'Waiting for mysqld to come online' - # The sleep 1 is there to make sure that inotifywait starts up before the socket is created - while [[ ! -S /var/run/mysqld/mysqld.sock ]]; do - sleep 1 - done - - echo 'Fixing root password' - /usr/bin/mysqladmin -u root password '' - - # if [ -d /var/lib/mysql/setup ]; then - # echo 'Found /var/lib/mysql/setup - scanning for SQL scripts' - # for sql in $(ls /var/lib/mysql/setup/*.sql 2>/dev/null | sort); do - # echo 'Running script:' $sql - # mysql -uroot -proot -e "\. $sql" - # mv $sql $sql.processed - # done - # else - # echo 'No setup directory with extra sql scripts to run' - # fi - - echo 'Shutting down mysqld' - mysqladmin -uroot shutdown - - retry=0 maxretry=10 - while [[ -e /var/run/mysqld/mysqld.sock && $retry -le $maxretry ]]; do - retry=$((retry+1)) - sleep 1 - done -fi diff --git a/image/base/services/memcached.sh b/image/base/services/memcached.sh deleted file mode 100755 index acebfdc..0000000 --- a/image/base/services/memcached.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# `/sbin/setuser memcache` runs the given command as the user `memcache`. -# If you omit that part, the command will be run as root. -exec /sbin/setuser memcache /usr/bin/memcached >>/var/log/memcached.log 2>&1 diff --git a/image/base/services/mysql.sh b/image/base/services/mysql.sh deleted file mode 100755 index 4b24039..0000000 --- a/image/base/services/mysql.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -shutdown_mysql() { - if [[ -S /var/run/mysqld/mysqld.sock ]]; then - mysqladmin -u root shutdown || true - fi -} - -trap shutdown_mysql EXIT - -mkdir -p /var/run/mysqld -chown mysql:mysql /var/run/mysqld - -rm -f /var/lib/mysql/aria_log_control - -/sbin/setuser mysql /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib/mysql/plugin --user=mysql --skip-log-error --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/run/mysqld/mysqld.sock --port=3306 >/var/log/mysql.log 2>&1 diff --git a/image/pro_base/Dockerfile b/image/pro_base/Dockerfile index d5fe177..7286358 100644 --- a/image/pro_base/Dockerfile +++ b/image/pro_base/Dockerfile @@ -22,4 +22,4 @@ RUN pip install boto==2.43.0 \ pycryptodome==3.7.2 \ twilio==5.7.0 -RUN apt clean +RUN apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index d01ac31..9e9ed39 100755 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -50,6 +50,9 @@ def init_letsencrypt(): loginfo('Found existing cert file {}'.format(ssl_crt)) if cert_has_valid_days(ssl_crt, 30): loginfo('Skip letsencrypt verification since we have a valid certificate') + if exists(join(ssl_dir, 'letsencrypt')): + # Create a crontab to auto renew the cert for letsencrypt. + call('/scripts/auto_renew_crt.sh {0} {1}'.format(ssl_dir, domain)) return loginfo('Starting letsencrypt verification') @@ -70,6 +73,9 @@ def init_letsencrypt(): # time.sleep(1000) # sys.exit(1) + call('/scripts/auto_renew_crt.sh {0} {1}'.format(ssl_dir, domain)) + # Create a crontab to auto renew the cert for letsencrypt. + def generate_local_nginx_conf(): # Now create the final nginx configuratin @@ -113,14 +119,22 @@ def init_seafile_server(): 'SERVER_IP': get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com'), 'MYSQL_USER': 'seafile', 'MYSQL_USER_PASSWD': str(uuid.uuid4()), - 'MYSQL_USER_HOST': '127.0.0.1', + 'MYSQL_USER_HOST': '%.%.%.%', + 'MYSQL_HOST': get_conf('DB_HOST','127.0.0.1'), # Default MariaDB root user has empty password and can only connect from localhost. - 'MYSQL_ROOT_PASSWD': '', + 'MYSQL_ROOT_PASSWD': get_conf('DB_ROOT_PASSWD', ''), } # Change the script to allow mysql root password to be empty - call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}''' - .format(get_script('setup-seafile-mysql.py'))) + # call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}''' + # .format(get_script('setup-seafile-mysql.py'))) + + # Change the script to disable check MYSQL_USER_HOST + call('''sed -i -e '/def validate_mysql_user_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}''' + .format(get_script('setup-seafile-mysql.py'))) + + call('''sed -i -e '/def validate_mysql_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}''' + .format(get_script('setup-seafile-mysql.py'))) setup_script = get_script('setup-seafile-mysql.sh') call('{} auto -n seafile'.format(setup_script), env=env) @@ -132,7 +146,7 @@ def init_seafile_server(): fp.write("""CACHES = { 'default': { 'BACKEND': 'django_pylibmc.memcached.PyLibMCCache', - 'LOCATION': '127.0.0.1:11211', + 'LOCATION': 'memcached:11211', }, 'locmem': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', diff --git a/scripts/create_data_links.sh b/scripts/create_data_links.sh index 547ebae..cdecc3c 100755 --- a/scripts/create_data_links.sh +++ b/scripts/create_data_links.sh @@ -39,12 +39,6 @@ if [[ ! -e /shared/logs/seafile ]]; then fi rm -rf /opt/seafile/logs && ln -sf /shared/logs/seafile/ /opt/seafile/logs -rm -rf /var/lib/mysql -if [[ ! -e /shared/db ]];then - mkdir -p /shared/db -fi -ln -sf /shared/db /var/lib/mysql - if [[ ! -e /shared/logs/var-log ]]; then mv /var/log /shared/logs/var-log fi diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index 222fee1..819d738 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -15,6 +15,7 @@ import logging.config import click import termcolor import colorlog +import MySQLdb logger = logging.getLogger('.utils') @@ -265,18 +266,19 @@ def update_version_stamp(version, fn=get_version_stamp_file()): fp.write(version + '\n') def wait_for_mysql(): - tried = False - while not exists('/var/run/mysqld/mysqld.sock'): - logdbg('waiting for mysql server to be ready') - """ - Try to solve the MySQL startup failure caused by the permission problem - caused by the MySQL user's uid change. - """ - if not tried: - os.system('rm /var/lib/mysql/tc.log -f && chown -R mysql.mysql /var/lib/mysql/') - tried = True - time.sleep(2) - logdbg('mysql server is ready') + db_host = get_conf('DB_HOST', '127.0.0.1') + db_user = 'root' + db_passwd = get_conf('DB_ROOT_PASSWD', '') + + while True: + try: + MySQLdb.connect(host=db_host, port=3306, user=db_user, passwd=db_passwd) + except Exception as e: + print ('waiting for mysql server to be ready: %s', e) + time.sleep(2) + continue + logdbg('mysql server is ready') + return def wait_for_nginx(): while True: