From e6a0840cf0958202bdadbecf13a81ee1a19cf529 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 14:44:26 +0800 Subject: [PATCH 1/7] Support manual major upgrade. * By default all major upgrades would be fully automatic. But the user can opt for manual upgrade by "./launcher rebuild --manual-upgrade" --- image/seafile/Dockerfile | 5 + launcher | 96 ++++++++++++++--- scripts/start.py | 10 +- scripts/upgrade.py | 82 +++++++++++++++ scripts/utils/__init__.py | 17 ++- tests/Dockerfile | 7 ++ tests/Makefile | 2 + tests/upgrade_6.0_6.1.sh | 210 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 410 insertions(+), 19 deletions(-) create mode 100755 scripts/upgrade.py create mode 100644 tests/Dockerfile create mode 100644 tests/Makefile create mode 100755 tests/upgrade_6.0_6.1.sh diff --git a/image/seafile/Dockerfile b/image/seafile/Dockerfile index b26f4dd..d2e7755 100644 --- a/image/seafile/Dockerfile +++ b/image/seafile/Dockerfile @@ -3,6 +3,11 @@ WORKDIR /opt/seafile ENV SEAFILE_VERSION=6.0.5 +# syslog-ng and syslog-forwarder would mess up the container stdout, not good +# when debugging/upgrading. +RUN sed -i -e 's|\(^exec syslog-ng.*$\)|\1 >>/var/log/syslog-ng.log 2>\&1|g' /etc/service/syslog-ng/run && \ + rm -rf /etc/service/syslog-forwarder + RUN mkdir -p /opt/seafile/ && \ curl -sSL -o - https://bintray.com/artifact/download/seafile-org/seafile/seafile-server_6.0.5_x86-64.tar.gz \ | tar xzf - -C /opt/seafile/ diff --git a/launcher b/launcher index 947789d..c5bd2ed 100755 --- a/launcher +++ b/launcher @@ -3,14 +3,14 @@ usage () { echo "Usage: launcher COMMAND [--skip-prereqs] [--docker-args STRING]" echo "Commands:" - echo " start: Start/initialize a container" + echo " start: Start/initialize the container" echo " stop: Stop a running container" - echo " restart: Restart a container" - echo " destroy: Stop and remove a container" + echo " restart: Restart the container" + echo " destroy: Stop and remove the container" echo " enter: Open a shell to run commands inside the container" - echo " logs: View the Docker logs for a container" - echo " bootstrap: Bootstrap a container for the config based on a template" - echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)" + echo " logs: View the Docker container logs" + echo " bootstrap: Bootstrap the container based on a template" + echo " rebuild: Rebuild the container (destroy old, bootstrap, start new)" echo echo "Options:" echo " --skip-prereqs Don't check launcher prerequisites" @@ -21,13 +21,14 @@ usage () { set -e set -o pipefail -version=6.0.5 +version=6.1.0 image=seafileorg/server:$version local_image=local_seafile/server:latest dockerdir=$(cd "$(dirname $0)"; pwd -P) sharedir=$dockerdir/shared installdir=/opt/seafile/seafile-server-$version bootstrap_conf=$dockerdir/bootstrap/bootstrap.conf +version_stamp_file=$sharedir/seafile/seafile-data/current_version cd $dockerdir @@ -154,6 +155,8 @@ start() { exit 0 fi + check_version_match + chmod 0700 $dockerdir/bootstrap $sharedir/seafile/conf set_existing_container @@ -240,6 +243,68 @@ destroy() { ) } +get_major_version() { + echo $1| awk -F . '{printf "%s.%s", $1, $2}' +} + +check_version_match() { + local last_version last_major_version current_major_version + last_version=$(cat $version_stamp_file) + last_major_version=$(get_major_version $last_version) + current_major_version=$(get_major_version $version) + + if [[ $last_major_version != "$current_major_version" ]]; then + show_progress "******* Major upgrade detected *******" + show_progress "You have $last_version, latest is $version" + show_progress "Please run './launcher rebuild' to upgrade" + exit 1 + fi +} + +check_upgrade() { + show_progress "Checking if there is major version upgrade" + + local last_version last_major_version current_major_version + last_version=$(cat $version_stamp_file) + last_major_version=$(get_major_version $last_version) + current_major_version=$(get_major_version $version) + + if [[ $last_major_version == "$current_major_version" ]]; then + return + else + show_progress "********************************" + show_progress "Major upgrade detected: You have $last_version, latest is $version" + show_progress "********************************" + + # use_manual_upgrade=true + if [[ $use_manual_upgrade == "true" ]]; then + show_progress "Now you can run './launcher manual-upgrade' to do manual upgrade." + exit 0 + else + show_progress "Going to launch the docker container for manual upgrade" + _launch_for_upgrade --auto + fi + fi +} + +_launch_for_upgrade() { + local cmd + if [[ $1 == "--auto" ]]; then + cmd="/scripts/upgrade.py" + else + cmd="/bin/bash" + fi + + set_volumes + ( + set -x + docker run $user_args \ + -it --rm --name seafile-upgrade -h seafile \ + $volumes $local_image \ + /sbin/my_init -- $cmd + ) +} + rebuild() { if [[ "$(git symbolic-ref --short HEAD)" == "master" ]]; then show_progress "Ensuring launcher is up to date" @@ -292,25 +357,32 @@ rebuild() { ) fi + check_upgrade + start show_progress "Your seafile server is now running." exit 0 } +manual_upgrade() { + _launch_for_upgrade +} + main() { local action while [[ $# -gt 0 ]] do case "$1" in - bootstrap|start|stop|restart|enter|destroy|logs|rebuild) - action=$1 ; shift 1 ;; - --debug) debug=true ; shift 1 ;; - --skip-prereqs) SKIP_PREREQS=true ; shift 1 ;; - --docker-args) user_args=$2 ; shift 2 ;; + bootstrap|start|stop|restart|enter|destroy|logs|rebuild|manual-upgrade) + action=${1//-/_} ; shift 1 ;; + --skip-prereqs) SKIP_PREREQS=true ; shift 1 ;; + --docker-args) user_args=$2 ; shift 2 ;; + --manual-upgrade) use_manual_upgrade=true ; shift 1 ;; *) err_and_quit "Argument error. Please see help." ;; esac done + "$action" } diff --git a/scripts/start.py b/scripts/start.py index fed22f5..65bdcb2 100755 --- a/scripts/start.py +++ b/scripts/start.py @@ -14,7 +14,10 @@ import shutil import sys import time -from utils import call, get_conf, get_install_dir, get_script, get_command_output, render_template +from utils import ( + call, get_conf, get_install_dir, get_script, get_command_output, + render_template, wait_for_mysql +) installdir = get_install_dir() topdir = dirname(installdir) @@ -39,10 +42,7 @@ def main(): with open(password_file, 'w') as fp: json.dump(admin_pw, fp) - while not exists('/var/run/mysqld/mysqld.sock'): - print 'waiting for mysql server to be ready' - time.sleep(2) - print 'mysql server is ready' + wait_for_mysql() try: call('{} start'.format(get_script('seafile.sh'))) diff --git a/scripts/upgrade.py b/scripts/upgrade.py new file mode 100755 index 0000000..516d4af --- /dev/null +++ b/scripts/upgrade.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +This script calls the appropriate seafile init scripts (e.g. +setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the +container. +""" + +import json +import re +import glob +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys +import time + +from utils import ( + call, get_install_dir, get_script, get_command_output, replace_file_pattern, + read_version_stamp, wait_for_mysql, update_version_stamp, show_progress +) + +installdir = get_install_dir() +topdir = dirname(installdir) + +def collect_upgrade_scripts(from_version, to_version): + """ + Give the current installed version, calculate which upgrade scripts we need + to run to upgrade it to the latest verison. + + For example, given current version 5.0.1 and target version 6.1.0, and these + upgrade scripts: + + upgrade_4.4_5.0.sh + upgrade_5.0_5.1.sh + upgrade_5.1_6.0.sh + upgrade_6.0_6.1.sh + + We need to run upgrade_5.0_5.1.sh, upgrade_5.1_6.0.sh, and upgrade_6.0_6.1.sh. + """ + from_major_ver = '.'.join(from_version.split('.')[:2]) + to_major_ver = '.'.join(to_version.split('.')[:2]) + + scripts = [] + for fn in glob.glob(join(installdir, 'upgrade', 'upgrade_*_*.sh')): + va, vb = parse_upgrade_script_version(fn) + if va >= from_major_ver and vb <= to_major_ver: + scripts.append(fn) + return scripts + +def parse_upgrade_script_version(script): + script = basename(script) + m = re.match(r'upgrade_([0-9+.]+)_([0-9+.]+).sh', basename(script)) + return m.groups() + +def check_upgrade(): + last_version = read_version_stamp() + current_version = os.environ['SEAFILE_VERSION'] + if last_version == current_version: + return + + scripts_to_run = collect_upgrade_scripts(from_version=last_version, to_version=current_version) + for script in scripts_to_run: + show_progress('Running scripts {}'.format(script)) + new_version = parse_upgrade_script_version(script)[1] + '.0' + + replace_file_pattern(script, 'read dummy', '') + call(script) + + update_version_stamp(new_version) + + update_version_stamp(current_version) + +def main(): + wait_for_mysql() + + os.chdir(installdir) + check_upgrade() + +if __name__ == '__main__': + main() diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index fc4eab8..9c53408 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -9,6 +9,7 @@ from os.path import abspath, basename, exists, dirname, join, isdir, expanduser import platform import sys import subprocess +import time import logging import logging.config import click @@ -251,11 +252,23 @@ def cert_has_valid_days(cert, days): def get_version_stamp_file(): return '/shared/seafile/seafile-data/current_version' -def read_version_stamp(fn): +def read_version_stamp(fn=get_version_stamp_file()): assert exists(fn), 'version stamp file {} does not exist!'.format(fn) with open(fn, 'r') as fp: return fp.read().strip() -def update_version_stamp(fn, version): +def update_version_stamp(version, fn=get_version_stamp_file()): with open(fn, 'w') as fp: fp.write(version + '\n') + +def wait_for_mysql(): + while not exists('/var/run/mysqld/mysqld.sock'): + print('waiting for mysql server to be ready') + time.sleep(2) + print('mysql server is ready') + +def replace_file_pattern(fn, pattern, replacement): + with open(fn, 'r') as fp: + content = fp.read() + with open(fn, 'w') as fp: + fp.write(content.replace(pattern, replacement)) diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..d386bb6 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,7 @@ +FROM seafileorg/server:6.0.5 + +ENV SEAFILE_VERSION=6.1.0 + +RUN mv /opt/seafile/seafile-server-6.0.5 /opt/seafile/seafile-server-${SEAFILE_VERSION} + +ADD upgrade_6.0_6.1.sh /opt/seafile/seafile-server-${SEAFILE_VERSION}/upgrade/upgrade_6.0_6.1.sh diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..685075f --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,2 @@ +all: + docker build -t seafileorg/server:6.1.0 . diff --git a/tests/upgrade_6.0_6.1.sh b/tests/upgrade_6.0_6.1.sh new file mode 100755 index 0000000..e858212 --- /dev/null +++ b/tests/upgrade_6.0_6.1.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +SCRIPT=$(readlink -f "$0") # haiwen/seafile-server-1.3.0/upgrade/upgrade_xx_xx.sh +UPGRADE_DIR=$(dirname "$SCRIPT") # haiwen/seafile-server-1.3.0/upgrade/ +INSTALLPATH=$(dirname "$UPGRADE_DIR") # haiwen/seafile-server-1.3.0/ +TOPDIR=$(dirname "${INSTALLPATH}") # haiwen/ +default_ccnet_conf_dir=${TOPDIR}/ccnet +default_conf_dir=${TOPDIR}/conf +seafile_server_symlink=${TOPDIR}/seafile-server-latest +seahub_data_dir=${TOPDIR}/seahub-data +seahub_settings_py=${TOPDIR}/seahub_settings.py + +manage_py=${INSTALLPATH}/seahub/manage.py + +export CCNET_CONF_DIR=${default_ccnet_conf_dir} +export SEAFILE_CENTRAL_CONF_DIR=${default_conf_dir} +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.6/site-packages:${INSTALLPATH}/seafile/lib64/python2.6/site-packages:${INSTALLPATH}/seafile/lib/python2.7/site-packages:${INSTALLPATH}/seahub/thirdpart:$PYTHONPATH +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.7/site-packages:${INSTALLPATH}/seafile/lib64/python2.7/site-packages:$PYTHONPATH +export SEAFILE_LD_LIBRARY_PATH=${INSTALLPATH}/seafile/lib/:${INSTALLPATH}/seafile/lib64:${LD_LIBRARY_PATH} + +prev_version=6.0 +current_version=6.1 + +echo +echo "-------------------------------------------------------------" +echo "This script would upgrade your seafile server from ${prev_version} to ${current_version}" +echo "Press [ENTER] to contiune" +echo "-------------------------------------------------------------" +echo +read dummy + +function check_python_executable() { + if [[ "$PYTHON" != "" && -x $PYTHON ]]; then + return 0 + fi + + if which python2.7 2>/dev/null 1>&2; then + PYTHON=python2.7 + elif which python27 2>/dev/null 1>&2; then + PYTHON=python27 + else + echo + echo "Can't find a python executable of version 2.7 or above in PATH" + echo "Install python 2.7+ before continue." + echo "Or if you installed it in a non-standard PATH, set the PYTHON enviroment varirable to it" + echo + exit 1 + fi +} + +function read_seafile_data_dir () { + seafile_ini=${default_ccnet_conf_dir}/seafile.ini + if [[ ! -f ${seafile_ini} ]]; then + echo "${seafile_ini} not found. Now quit" + exit 1 + fi + seafile_data_dir=$(cat "${seafile_ini}") + if [[ ! -d ${seafile_data_dir} ]]; then + echo "Your seafile server data directory \"${seafile_data_dir}\" is invalid or doesn't exits." + echo "Please check it first, or create this directory yourself." + echo "" + exit 1; + fi + + export SEAFILE_CONF_DIR=$seafile_data_dir +} + +function ensure_server_not_running() { + # test whether seafile server has been stopped. + if pgrep seaf-server 2>/dev/null 1>&2 ; then + echo + echo "seafile server is still running !" + echo "stop it using scripts before upgrade." + echo + exit 1 + elif pgrep -f "${manage_py} run_gunicorn" 2>/dev/null 1>&2 \ + || pgrep -f "seahub.wsgi:application" 2>/dev/null 1>&2; then + echo + echo "seahub server is still running !" + echo "stop it before upgrade." + echo + exit 1 + elif pgrep -f "${manage_py} runfcgi" 2>/dev/null 1>&2 ; then + echo + echo "seahub server is still running !" + echo "stop it before upgrade." + echo + exit 1 + fi +} + +function migrate_avatars() { + echo + echo "migrating avatars ..." + echo + media_dir=${INSTALLPATH}/seahub/media + orig_avatar_dir=${INSTALLPATH}/seahub/media/avatars + dest_avatar_dir=${TOPDIR}/seahub-data/avatars + + # move "media/avatars" directory outside + if [[ ! -d ${dest_avatar_dir} ]]; then + mkdir -p "${TOPDIR}/seahub-data" + mv "${orig_avatar_dir}" "${dest_avatar_dir}" 2>/dev/null 1>&2 + ln -s ../../../seahub-data/avatars "${media_dir}" + + elif [[ ! -L ${orig_avatar_dir} ]]; then + mv "${orig_avatar_dir}"/* "${dest_avatar_dir}" 2>/dev/null 1>&2 + rm -rf "${orig_avatar_dir}" + ln -s ../../../seahub-data/avatars "${media_dir}" + fi + echo "Done" +} + +function update_database() { + echo + echo "Updating seafile/seahub database ..." + echo + + db_update_helper=${UPGRADE_DIR}/db_update_helper.py + if ! $PYTHON "${db_update_helper}" 6.0.0; then + echo + echo "Failed to upgrade your database" + echo + exit 1 + fi + echo "Done" +} + +function upgrade_seafile_server_latest_symlink() { + # update the symlink seafile-server to the new server version + if [[ -L "${seafile_server_symlink}" || ! -e "${seafile_server_symlink}" ]]; then + echo + printf "updating \033[33m${seafile_server_symlink}\033[m symbolic link to \033[33m${INSTALLPATH}\033[m ...\n\n" + echo + if ! rm -f "${seafile_server_symlink}"; then + echo "Failed to remove ${seafile_server_symlink}" + echo + exit 1; + fi + + if ! ln -s "$(basename ${INSTALLPATH})" "${seafile_server_symlink}"; then + echo "Failed to update ${seafile_server_symlink} symbolic link." + echo + exit 1; + fi + fi +} + +function make_media_custom_symlink() { + media_symlink=${INSTALLPATH}/seahub/media/custom + if [[ -L "${media_symlink}" ]]; then + return + + elif [[ ! -e "${media_symlink}" ]]; then + ln -s ../../../seahub-data/custom "${media_symlink}" + return + + + elif [[ -d "${media_symlink}" ]]; then + cp -rf "${media_symlink}" "${seahub_data_dir}/" + rm -rf "${media_symlink}" + ln -s ../../../seahub-data/custom "${media_symlink}" + fi + +} + +function move_old_customdir_outside() { + # find the path of the latest seafile server folder + if [[ -L ${seafile_server_symlink} ]]; then + latest_server=$(readlink -f "${seafile_server_symlink}") + else + return + fi + + old_customdir=${latest_server}/seahub/media/custom + + # old customdir is already a symlink, do nothing + if [[ -L "${old_customdir}" ]]; then + return + fi + + # old customdir does not exist, do nothing + if [[ ! -e "${old_customdir}" ]]; then + return + fi + + # media/custom exist and is not a symlink + cp -rf "${old_customdir}" "${seahub_data_dir}/" +} + +################# +# The main execution flow of the script +################ + +check_python_executable; +read_seafile_data_dir; +ensure_server_not_running; + +# update_database; +migrate_avatars; + +move_old_customdir_outside; +make_media_custom_symlink; +upgrade_seafile_server_latest_symlink; + +echo +echo "-----------------------------------------------------------------" +echo "Upgraded your seafile server successfully." +echo "-----------------------------------------------------------------" +echo From 12412e733d4aaf6dc064ab5796beae331c874e23 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 15:09:16 +0800 Subject: [PATCH 2/7] Suppport verbose mode: "./launcher -v " --- launcher | 37 +++++++++++++++++++++++++++++-------- scripts/bootstrap.py | 9 +++++---- scripts/utils/__init__.py | 9 +++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/launcher b/launcher index c5bd2ed..ff42ff6 100755 --- a/launcher +++ b/launcher @@ -21,7 +21,7 @@ usage () { set -e set -o pipefail -version=6.1.0 +version=6.0.5 image=seafileorg/server:$version local_image=local_seafile/server:latest dockerdir=$(cd "$(dirname $0)"; pwd -P) @@ -32,9 +32,9 @@ version_stamp_file=$sharedir/seafile/seafile-data/current_version cd $dockerdir -dbg() { - if [[ $debug == "true" ]]; then - echo "dbg: $1" +dlog() { + if [[ $verbose == "true" ]]; then + show_progress "[debug] $1" fi } @@ -76,6 +76,13 @@ init_shared() { fi } +set_envs() { + envs="" + if [[ $verbose == "true" ]]; then + envs="$envs -e SEAFILE_DOCKER_VERBOSE=true" + fi +} + set_ports() { ports=$(docker run $user_args --rm -it \ -v ${dockerdir}/scripts:/scripts \ @@ -138,10 +145,11 @@ bootstrap() { } # First initialize seafile server and letsencrypt + set_envs set_bootstrap_volumes set_ports - docker run $user_args --rm -it --name seafile-bootstrap -e SEAFILE_BOOTSRAP=1 $volumes $ports $image /sbin/my_init -- /scripts/bootstrap.py + docker run $user_args --rm -it --name seafile-bootstrap -e SEAFILE_BOOTSRAP=1 $envs $volumes $ports $image /sbin/my_init $quiet -- /scripts/bootstrap.py show_progress "Now building the local docker image." docker build -f bootstrap/generated/Dockerfile -t local_seafile/server:latest . >/dev/null @@ -169,6 +177,7 @@ start() { exit 0 fi + set_envs set_volumes set_ports @@ -183,7 +192,7 @@ start() { show_progress "Starting up new seafile server container" ( set -x - docker run $user_args $attach_on_run $restart_policy --name seafile -h seafile $volumes $ports $local_image + docker run $user_args $attach_on_run $restart_policy --name seafile -h seafile $envs $volumes $ports $local_image ) } @@ -253,6 +262,8 @@ check_version_match() { last_major_version=$(get_major_version $last_version) current_major_version=$(get_major_version $version) + dlog "Your version: ${last_version}, latest version: ${version}" + if [[ $last_major_version != "$current_major_version" ]]; then show_progress "******* Major upgrade detected *******" show_progress "You have $last_version, latest is $version" @@ -295,13 +306,14 @@ _launch_for_upgrade() { cmd="/bin/bash" fi + set_envs set_volumes ( set -x docker run $user_args \ -it --rm --name seafile-upgrade -h seafile \ - $volumes $local_image \ - /sbin/my_init -- $cmd + $envs $volumes $local_image \ + /sbin/my_init $quiet -- $cmd ) } @@ -376,6 +388,8 @@ main() { case "$1" in bootstrap|start|stop|restart|enter|destroy|logs|rebuild|manual-upgrade) action=${1//-/_} ; shift 1 ;; + -h|--help) ( usage ; exit 1) ; shift 1 ;; + -v|--verbose) verbose=true ; shift 1 ;; --skip-prereqs) SKIP_PREREQS=true ; shift 1 ;; --docker-args) user_args=$2 ; shift 2 ;; --manual-upgrade) use_manual_upgrade=true ; shift 1 ;; @@ -383,6 +397,13 @@ main() { esac done + # By default we suppress the verbose logs like "Running + # /etc/my_init.d/99_mysql_setup.sh". Use "./launcher -v " to enable + # printing of these verbose logs. + quiet="--quiet" + if [[ $verbose == "true" ]]; then + quiet="" + fi "$action" } diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index a2ea919..d9689c3 100755 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -18,7 +18,8 @@ import time from utils import ( call, get_conf, get_install_dir, show_progress, get_script, render_template, get_seafile_version, eprint, - cert_has_valid_days, get_version_stamp_file, update_version_stamp + cert_has_valid_days, get_version_stamp_file, update_version_stamp, + wait_for_mysql, wait_for_nginx ) seafile_version = get_seafile_version() @@ -61,9 +62,7 @@ def init_letsencrypt(): render_template('/templates/seafile.nginx.conf.template', '/etc/nginx/sites-enabled/seafile.nginx.conf', context) - # TODO: The 5 seconds heuristic is not good, how can we know for sure nginx is ready? - print 'waiting for nginx server to be ready' - time.sleep(5) + wait_for_nginx() call('nginx -s reload') call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain)) @@ -156,6 +155,7 @@ def init_seafile_server(): if not exists(dst) and exists(src): shutil.move(src, shared_seafiledir) + show_progress('Updating version stamp') update_version_stamp(version_stamp_file, os.environ['SEAFILE_VERSION']) def main(): @@ -174,6 +174,7 @@ def main(): init_letsencrypt() generate_local_nginx_conf() + wait_for_mysql() init_seafile_server() show_progress('Generated local config.') diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index 9c53408..dd070d9 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -267,6 +267,15 @@ def wait_for_mysql(): time.sleep(2) print('mysql server is ready') +def wait_for_nginx(): + while True: + print('waiting for nginx server to be ready') + output = get_command_output('netstat -nltp') + if ':80 ' in output: + print('nginx is ready') + return + time.sleep(2) + def replace_file_pattern(fn, pattern, replacement): with open(fn, 'r') as fp: content = fp.read() From cc77e95d9fb362534b31a48bb5a06cff7866c09b Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 15:26:33 +0800 Subject: [PATCH 3/7] Make debug logs and info logs more clear. --- image/base/services/nginx.conf | 7 ++++ launcher | 58 +++++++++++++++++----------------- scripts/bootstrap.py | 27 ++++++++-------- scripts/upgrade.py | 4 +-- scripts/utils/__init__.py | 22 +++++++++---- 5 files changed, 67 insertions(+), 51 deletions(-) diff --git a/image/base/services/nginx.conf b/image/base/services/nginx.conf index 1bab6b5..c2a6b12 100644 --- a/image/base/services/nginx.conf +++ b/image/base/services/nginx.conf @@ -23,4 +23,11 @@ http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/sites-enabled/*; + + server { + listen 80; + location / { + return 444; + } + } } diff --git a/launcher b/launcher index ff42ff6..b79b385 100755 --- a/launcher +++ b/launcher @@ -32,13 +32,13 @@ version_stamp_file=$sharedir/seafile/seafile-data/current_version cd $dockerdir -dlog() { +logdbg() { if [[ $verbose == "true" ]]; then - show_progress "[debug] $1" + loginfo "[debug] $1" fi } -show_progress() { +loginfo() { if [[ -t 1 ]]; then >&2 printf "[$(date +'%Y-%m-%d %H:%M:%S')] \033[32m%s\033[m\n" "$1" else @@ -139,9 +139,9 @@ bootstrap() { fi docker history $image >/dev/null 2>&1 || { - show_progress "Pulling Seafile server image $version, this may take a while." + loginfo "Pulling Seafile server image $version, this may take a while." docker pull $image - show_progress "Seafile server image $version pulled. Now bootstrapping the server ..." + loginfo "Seafile server image $version pulled. Now bootstrapping the server ..." } # First initialize seafile server and letsencrypt @@ -151,15 +151,15 @@ bootstrap() { docker run $user_args --rm -it --name seafile-bootstrap -e SEAFILE_BOOTSRAP=1 $envs $volumes $ports $image /sbin/my_init $quiet -- /scripts/bootstrap.py - show_progress "Now building the local docker image." + loginfo "Now building the local docker image." docker build -f bootstrap/generated/Dockerfile -t local_seafile/server:latest . >/dev/null - show_progress "Image built." + loginfo "Image built." } start() { existing=$(docker ps | awk '{ print $1, $(NF) }' | grep " seafile$" | awk '{ print $1 }' || true) if [[ $existing != "" ]]; then - show_progress "Nothing to do, your container has already started!" + loginfo "Nothing to do, your container has already started!" exit 0 fi @@ -169,7 +169,7 @@ start() { set_existing_container if [[ $existing != "" ]]; then - show_progress "starting up existing container" + loginfo "starting up existing container" ( set -x docker start seafile @@ -189,7 +189,7 @@ start() { attach_on_run="-d" fi - show_progress "Starting up new seafile server container" + loginfo "Starting up new seafile server container" ( set -x docker run $user_args $attach_on_run $restart_policy --name seafile -h seafile $envs $volumes $ports $local_image @@ -262,18 +262,18 @@ check_version_match() { last_major_version=$(get_major_version $last_version) current_major_version=$(get_major_version $version) - dlog "Your version: ${last_version}, latest version: ${version}" + logdbg "Your version: ${last_version}, latest version: ${version}" if [[ $last_major_version != "$current_major_version" ]]; then - show_progress "******* Major upgrade detected *******" - show_progress "You have $last_version, latest is $version" - show_progress "Please run './launcher rebuild' to upgrade" + loginfo "******* Major upgrade detected *******" + loginfo "You have $last_version, latest is $version" + loginfo "Please run './launcher rebuild' to upgrade" exit 1 fi } check_upgrade() { - show_progress "Checking if there is major version upgrade" + loginfo "Checking if there is major version upgrade" local last_version last_major_version current_major_version last_version=$(cat $version_stamp_file) @@ -283,16 +283,16 @@ check_upgrade() { if [[ $last_major_version == "$current_major_version" ]]; then return else - show_progress "********************************" - show_progress "Major upgrade detected: You have $last_version, latest is $version" - show_progress "********************************" + loginfo "********************************" + loginfo "Major upgrade detected: You have $last_version, latest is $version" + loginfo "********************************" # use_manual_upgrade=true if [[ $use_manual_upgrade == "true" ]]; then - show_progress "Now you can run './launcher manual-upgrade' to do manual upgrade." + loginfo "Now you can run './launcher manual-upgrade' to do manual upgrade." exit 0 else - show_progress "Going to launch the docker container for manual upgrade" + loginfo "Going to launch the docker container for manual upgrade" _launch_for_upgrade --auto fi fi @@ -319,7 +319,7 @@ _launch_for_upgrade() { rebuild() { if [[ "$(git symbolic-ref --short HEAD)" == "master" ]]; then - show_progress "Ensuring launcher is up to date" + loginfo "Ensuring launcher is up to date" git remote update @@ -328,10 +328,10 @@ rebuild() { BASE=$(git merge-base @ "@{u}") if [[ $LOCAL = "$REMOTE" ]]; then - show_progress "Launcher is up-to-date" + loginfo "Launcher is up-to-date" elif [[ $LOCAL = "$BASE" ]]; then - show_progress "Updating Launcher" + loginfo "Updating Launcher" git pull || (echo 'failed to update' && exit 1) for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ )) @@ -341,9 +341,9 @@ rebuild() { exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead. elif [[ $REMOTE = "$BASE" ]]; then - show_progress "Your version of Launcher is ahead of origin" + loginfo "Your version of Launcher is ahead of origin" else - show_progress "Launcher has diverged source, this is only expected in Dev mode" + loginfo "Launcher has diverged source, this is only expected in Dev mode" fi fi @@ -351,7 +351,7 @@ rebuild() { set_existing_container if [[ $existing != "" ]]; then - show_progress "Stopping old container" + loginfo "Stopping old container" ( set -x docker stop -t 10 seafile @@ -359,10 +359,10 @@ rebuild() { fi bootstrap - show_progress "Rebuilt successfully." + loginfo "Rebuilt successfully." if [[ $existing != "" ]]; then - show_progress "Removing old container" + loginfo "Removing old container" ( set -x docker rm seafile @@ -373,7 +373,7 @@ rebuild() { start - show_progress "Your seafile server is now running." + loginfo "Your seafile server is now running." exit 0 } diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index d9689c3..300a1b2 100755 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -16,7 +16,7 @@ import uuid import time from utils import ( - call, get_conf, get_install_dir, show_progress, + call, get_conf, get_install_dir, loginfo, get_script, render_template, get_seafile_version, eprint, cert_has_valid_days, get_version_stamp_file, update_version_stamp, wait_for_mysql, wait_for_nginx @@ -30,12 +30,13 @@ ssl_dir = '/shared/ssl' generated_dir = '/bootstrap/generated' def init_letsencrypt(): - show_progress('Preparing for letsencrypt ...') + loginfo('Preparing for letsencrypt ...') + wait_for_nginx() + if not exists(ssl_dir): os.mkdir(ssl_dir) domain = get_conf('server.hostname') - context = { 'ssl_dir': ssl_dir, 'domain': domain, @@ -48,12 +49,12 @@ def init_letsencrypt(): ssl_crt = '/shared/ssl/{}.crt'.format(domain) if exists(ssl_crt): - show_progress('Found existing cert file {}'.format(ssl_crt)) + loginfo('Found existing cert file {}'.format(ssl_crt)) if cert_has_valid_days(ssl_crt, 30): - show_progress('Skip letsencrypt verification since we have a valid certificate') + loginfo('Skip letsencrypt verification since we have a valid certificate') return - show_progress('Starting letsencrypt verification') + loginfo('Starting letsencrypt verification') # Create a temporary nginx conf to start a server, which would accessed by letsencrypt context = { 'https': False, @@ -62,8 +63,8 @@ def init_letsencrypt(): render_template('/templates/seafile.nginx.conf.template', '/etc/nginx/sites-enabled/seafile.nginx.conf', context) - wait_for_nginx() call('nginx -s reload') + time.sleep(2) call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain)) # if call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain), check_call=False) != 0: @@ -90,7 +91,7 @@ def is_https(): return get_conf('server.letsencrypt', '').lower() == 'true' def generate_local_dockerfile(): - show_progress('Generating local Dockerfile ...') + loginfo('Generating local Dockerfile ...') context = { 'seafile_version': seafile_version, 'https': is_https(), @@ -120,10 +121,10 @@ def init_seafile_server(): if exists(join(shared_seafiledir, 'seafile-data')): if not exists(version_stamp_file): update_version_stamp(version_stamp_file, os.environ['SEAFILE_VERSION']) - show_progress('Skip running setup-seafile-mysql.py because there is existing seafile-data folder.') + loginfo('Skip running setup-seafile-mysql.py because there is existing seafile-data folder.') return - show_progress('Now running setup-seafile-mysql.py in auto mode.') + loginfo('Now running setup-seafile-mysql.py in auto mode.') env = { 'SERVER_NAME': 'seafile', 'SERVER_IP': get_conf('server.hostname'), @@ -155,8 +156,8 @@ def init_seafile_server(): if not exists(dst) and exists(src): shutil.move(src, shared_seafiledir) - show_progress('Updating version stamp') - update_version_stamp(version_stamp_file, os.environ['SEAFILE_VERSION']) + loginfo('Updating version stamp') + update_version_stamp(os.environ['SEAFILE_VERSION']) def main(): args = parse_args() @@ -177,7 +178,7 @@ def main(): wait_for_mysql() init_seafile_server() - show_progress('Generated local config.') + loginfo('Generated local config.') if __name__ == '__main__': diff --git a/scripts/upgrade.py b/scripts/upgrade.py index 516d4af..ec9bd48 100755 --- a/scripts/upgrade.py +++ b/scripts/upgrade.py @@ -18,7 +18,7 @@ import time from utils import ( call, get_install_dir, get_script, get_command_output, replace_file_pattern, - read_version_stamp, wait_for_mysql, update_version_stamp, show_progress + read_version_stamp, wait_for_mysql, update_version_stamp, loginfo ) installdir = get_install_dir() @@ -62,7 +62,7 @@ def check_upgrade(): scripts_to_run = collect_upgrade_scripts(from_version=last_version, to_version=current_version) for script in scripts_to_run: - show_progress('Running scripts {}'.format(script)) + loginfo('Running scripts {}'.format(script)) new_version = parse_upgrade_script_version(script)[1] + '.0' replace_file_pattern(script, 'read dummy', '') diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index dd070d9..42d06af 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -58,8 +58,8 @@ def call(*a, **kw): for arg in reduct_args: value = _find_flag(args, arg) toprint = toprint.replace(value, '{}**reducted**'.format(value[:3])) - eprint('calling: ', green(toprint)) - eprint('cwd: ', green(cwd)) + logdbg('calling: ' + green(toprint)) + logdbg('cwd: ' + green(cwd)) kw.setdefault('shell', True) if not dry_run: if check_call: @@ -238,7 +238,14 @@ def render_template(template, target, context): with open(target, 'w') as fp: fp.write(content) -def show_progress(msg): +def logdbg(msg): + if DEBUG_ENABLED: + msg = '[debug] ' + msg + loginfo(msg) + +DEBUG_ENABLED = os.environ.get('SEAFILE_DOCKER_VERBOSE', '').lower() in ('true', '1', 'yes') + +def loginfo(msg): msg = '[{}] {}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), green(msg)) eprint(msg) @@ -263,16 +270,17 @@ def update_version_stamp(version, fn=get_version_stamp_file()): def wait_for_mysql(): while not exists('/var/run/mysqld/mysqld.sock'): - print('waiting for mysql server to be ready') + logdbg('waiting for mysql server to be ready') time.sleep(2) - print('mysql server is ready') + logdbg('mysql server is ready') def wait_for_nginx(): while True: - print('waiting for nginx server to be ready') + logdbg('waiting for nginx server to be ready') output = get_command_output('netstat -nltp') if ':80 ' in output: - print('nginx is ready') + logdbg(output) + logdbg('nginx is ready') return time.sleep(2) From e2557cd45ae48d0fcf20c9a9b61fc191001dcdfa Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 15:29:36 +0800 Subject: [PATCH 4/7] Updated README. --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ea4e026..2982fc8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ sudo ./launcher restart The logs are under `shared/logs/seafile`. +#### Enable debug logs of launcher + +Pass `-v` to launcher script (for example `sudo ./launcher -v rebuild` would make it print more verbose information + ### Directory Structure @@ -115,13 +119,13 @@ If you want to use your own SSL certificate: ### Upgrading Seafile Server -Simple run `./launcher rebuild`, which would keep your seafile server up to date. +Simple run `sudo ./launcher rebuild`, which would keep your seafile server up to date. ### Troubleshooting -View the container logs: `./launcher logs` +View the container logs: `sudo ./launcher logs` -Spawn a shell inside your container using `./launcher enter`. This is the most foolproof method if you have host root access. +Spawn a shell inside your container using `sudo ./launcher enter`. This is the most foolproof method if you have host root access. ### Developing with Vagrant From ccedb31221cec12b23ab64e16a9810d8410fc6f3 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 15:32:21 +0800 Subject: [PATCH 5/7] More tests in travis builds. --- ci/ci.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/ci.sh b/ci/ci.sh index 901b1dd..f1a8d6e 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -11,9 +11,10 @@ pip install docker-squash sudo cp samples/server.conf bootstrap/bootstrap.conf -sudo ./launcher bootstrap -sudo ./launcher start && sleep 10 +sudo ./launcher -v bootstrap +sudo ./launcher -v start && sleep 10 sudo ./launcher stop --skip-prereqs sudo ./launcher start --docker-args "--memory 1g" && sleep 10 sudo ./launcher restart -sudo ./launcher rebuild --docker-args "--memory 1g" +sudo ./launcher -v rebuild +sudo ./launcher -v rebuild --docker-args "--memory 1g" From ea4f3762f7d34eab50508fcc52aff0665d3d5cc9 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 17:09:11 +0800 Subject: [PATCH 6/7] Fixed support manual major upgrade. --- launcher | 5 +++++ scripts/utils/__init__.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/launcher b/launcher index b79b385..d751e65 100755 --- a/launcher +++ b/launcher @@ -379,6 +379,11 @@ rebuild() { manual_upgrade() { _launch_for_upgrade + loginfo "If you have manually upgraded the server, please update the version stamp by:" + loginfo + loginfo " echo $version | sudo tee $version_stamp_file" + loginfo " sudo ./launcher start" + loginfo } main() { diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index 42d06af..f899070 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -18,6 +18,8 @@ import colorlog logger = logging.getLogger('.utils') +DEBUG_ENABLED = os.environ.get('SEAFILE_DOCKER_VERBOSE', '').lower() in ('true', '1', 'yes') + def eprint(*a, **kw): kw['file'] = sys.stderr print(*a, **kw) @@ -48,7 +50,7 @@ def _find_flag(args, *opts, **kw): def call(*a, **kw): dry_run = kw.pop('dry_run', False) - quiet = kw.pop('quiet', False) + quiet = kw.pop('quiet', DEBUG_ENABLED) cwd = kw.get('cwd', os.getcwd()) check_call = kw.pop('check_call', True) reduct_args = kw.pop('reduct_args', []) @@ -243,8 +245,6 @@ def logdbg(msg): msg = '[debug] ' + msg loginfo(msg) -DEBUG_ENABLED = os.environ.get('SEAFILE_DOCKER_VERBOSE', '').lower() in ('true', '1', 'yes') - def loginfo(msg): msg = '[{}] {}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), green(msg)) eprint(msg) From 01d1035433d65aa7010c46048fe3687c38f36ba0 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 17:13:06 +0800 Subject: [PATCH 7/7] Improve python scripts docstring. --- scripts/bootstrap.py | 4 +--- scripts/start.py | 9 +++++---- scripts/upgrade.py | 4 +--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index 300a1b2..bd588c8 100755 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -2,9 +2,7 @@ #coding: UTF-8 """ -This script calls the appropriate seafile init scripts (e.g. -setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the -container. +Bootstraping seafile server, letsencrypt (verification & cron job). """ import argparse diff --git a/scripts/start.py b/scripts/start.py index 65bdcb2..913638f 100755 --- a/scripts/start.py +++ b/scripts/start.py @@ -2,9 +2,8 @@ #coding: UTF-8 """ -This script calls the appropriate seafile init scripts (e.g. -setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the -container. +Starts the seafile/seahub server and watches the controller process. It is +the entrypoint command of the docker container. """ import json @@ -29,7 +28,9 @@ def watch_controller(): controller_pid = get_command_output('ps aux | grep seafile-controller |grep -v grep || true').strip() if not controller_pid: retry += 1 - time.sleep(2) + else: + retry = 0 + time.sleep(5) print 'seafile controller exited unexpectedly.' sys.exit(1) diff --git a/scripts/upgrade.py b/scripts/upgrade.py index ec9bd48..fbc6b50 100755 --- a/scripts/upgrade.py +++ b/scripts/upgrade.py @@ -2,9 +2,7 @@ #coding: UTF-8 """ -This script calls the appropriate seafile init scripts (e.g. -setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the -container. +This script is used to run proper upgrade scripts automatically. """ import json