From e6a0840cf0958202bdadbecf13a81ee1a19cf529 Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Tue, 29 Nov 2016 14:44:26 +0800 Subject: [PATCH] Support manual major upgrade. * By default all major upgrades would be fully automatic. But the user can opt for manual upgrade by "./launcher rebuild --manual-upgrade" --- image/seafile/Dockerfile | 5 + launcher | 96 ++++++++++++++--- scripts/start.py | 10 +- scripts/upgrade.py | 82 +++++++++++++++ scripts/utils/__init__.py | 17 ++- tests/Dockerfile | 7 ++ tests/Makefile | 2 + tests/upgrade_6.0_6.1.sh | 210 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 410 insertions(+), 19 deletions(-) create mode 100755 scripts/upgrade.py create mode 100644 tests/Dockerfile create mode 100644 tests/Makefile create mode 100755 tests/upgrade_6.0_6.1.sh diff --git a/image/seafile/Dockerfile b/image/seafile/Dockerfile index b26f4dd..d2e7755 100644 --- a/image/seafile/Dockerfile +++ b/image/seafile/Dockerfile @@ -3,6 +3,11 @@ WORKDIR /opt/seafile ENV SEAFILE_VERSION=6.0.5 +# syslog-ng and syslog-forwarder would mess up the container stdout, not good +# when debugging/upgrading. +RUN sed -i -e 's|\(^exec syslog-ng.*$\)|\1 >>/var/log/syslog-ng.log 2>\&1|g' /etc/service/syslog-ng/run && \ + rm -rf /etc/service/syslog-forwarder + RUN mkdir -p /opt/seafile/ && \ curl -sSL -o - https://bintray.com/artifact/download/seafile-org/seafile/seafile-server_6.0.5_x86-64.tar.gz \ | tar xzf - -C /opt/seafile/ diff --git a/launcher b/launcher index 947789d..c5bd2ed 100755 --- a/launcher +++ b/launcher @@ -3,14 +3,14 @@ usage () { echo "Usage: launcher COMMAND [--skip-prereqs] [--docker-args STRING]" echo "Commands:" - echo " start: Start/initialize a container" + echo " start: Start/initialize the container" echo " stop: Stop a running container" - echo " restart: Restart a container" - echo " destroy: Stop and remove a container" + echo " restart: Restart the container" + echo " destroy: Stop and remove the container" echo " enter: Open a shell to run commands inside the container" - echo " logs: View the Docker logs for a container" - echo " bootstrap: Bootstrap a container for the config based on a template" - echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)" + echo " logs: View the Docker container logs" + echo " bootstrap: Bootstrap the container based on a template" + echo " rebuild: Rebuild the container (destroy old, bootstrap, start new)" echo echo "Options:" echo " --skip-prereqs Don't check launcher prerequisites" @@ -21,13 +21,14 @@ usage () { set -e set -o pipefail -version=6.0.5 +version=6.1.0 image=seafileorg/server:$version local_image=local_seafile/server:latest dockerdir=$(cd "$(dirname $0)"; pwd -P) sharedir=$dockerdir/shared installdir=/opt/seafile/seafile-server-$version bootstrap_conf=$dockerdir/bootstrap/bootstrap.conf +version_stamp_file=$sharedir/seafile/seafile-data/current_version cd $dockerdir @@ -154,6 +155,8 @@ start() { exit 0 fi + check_version_match + chmod 0700 $dockerdir/bootstrap $sharedir/seafile/conf set_existing_container @@ -240,6 +243,68 @@ destroy() { ) } +get_major_version() { + echo $1| awk -F . '{printf "%s.%s", $1, $2}' +} + +check_version_match() { + local last_version last_major_version current_major_version + last_version=$(cat $version_stamp_file) + last_major_version=$(get_major_version $last_version) + current_major_version=$(get_major_version $version) + + if [[ $last_major_version != "$current_major_version" ]]; then + show_progress "******* Major upgrade detected *******" + show_progress "You have $last_version, latest is $version" + show_progress "Please run './launcher rebuild' to upgrade" + exit 1 + fi +} + +check_upgrade() { + show_progress "Checking if there is major version upgrade" + + local last_version last_major_version current_major_version + last_version=$(cat $version_stamp_file) + last_major_version=$(get_major_version $last_version) + current_major_version=$(get_major_version $version) + + if [[ $last_major_version == "$current_major_version" ]]; then + return + else + show_progress "********************************" + show_progress "Major upgrade detected: You have $last_version, latest is $version" + show_progress "********************************" + + # use_manual_upgrade=true + if [[ $use_manual_upgrade == "true" ]]; then + show_progress "Now you can run './launcher manual-upgrade' to do manual upgrade." + exit 0 + else + show_progress "Going to launch the docker container for manual upgrade" + _launch_for_upgrade --auto + fi + fi +} + +_launch_for_upgrade() { + local cmd + if [[ $1 == "--auto" ]]; then + cmd="/scripts/upgrade.py" + else + cmd="/bin/bash" + fi + + set_volumes + ( + set -x + docker run $user_args \ + -it --rm --name seafile-upgrade -h seafile \ + $volumes $local_image \ + /sbin/my_init -- $cmd + ) +} + rebuild() { if [[ "$(git symbolic-ref --short HEAD)" == "master" ]]; then show_progress "Ensuring launcher is up to date" @@ -292,25 +357,32 @@ rebuild() { ) fi + check_upgrade + start show_progress "Your seafile server is now running." exit 0 } +manual_upgrade() { + _launch_for_upgrade +} + main() { local action while [[ $# -gt 0 ]] do case "$1" in - bootstrap|start|stop|restart|enter|destroy|logs|rebuild) - action=$1 ; shift 1 ;; - --debug) debug=true ; shift 1 ;; - --skip-prereqs) SKIP_PREREQS=true ; shift 1 ;; - --docker-args) user_args=$2 ; shift 2 ;; + bootstrap|start|stop|restart|enter|destroy|logs|rebuild|manual-upgrade) + action=${1//-/_} ; shift 1 ;; + --skip-prereqs) SKIP_PREREQS=true ; shift 1 ;; + --docker-args) user_args=$2 ; shift 2 ;; + --manual-upgrade) use_manual_upgrade=true ; shift 1 ;; *) err_and_quit "Argument error. Please see help." ;; esac done + "$action" } diff --git a/scripts/start.py b/scripts/start.py index fed22f5..65bdcb2 100755 --- a/scripts/start.py +++ b/scripts/start.py @@ -14,7 +14,10 @@ import shutil import sys import time -from utils import call, get_conf, get_install_dir, get_script, get_command_output, render_template +from utils import ( + call, get_conf, get_install_dir, get_script, get_command_output, + render_template, wait_for_mysql +) installdir = get_install_dir() topdir = dirname(installdir) @@ -39,10 +42,7 @@ def main(): with open(password_file, 'w') as fp: json.dump(admin_pw, fp) - while not exists('/var/run/mysqld/mysqld.sock'): - print 'waiting for mysql server to be ready' - time.sleep(2) - print 'mysql server is ready' + wait_for_mysql() try: call('{} start'.format(get_script('seafile.sh'))) diff --git a/scripts/upgrade.py b/scripts/upgrade.py new file mode 100755 index 0000000..516d4af --- /dev/null +++ b/scripts/upgrade.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +This script calls the appropriate seafile init scripts (e.g. +setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the +container. +""" + +import json +import re +import glob +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys +import time + +from utils import ( + call, get_install_dir, get_script, get_command_output, replace_file_pattern, + read_version_stamp, wait_for_mysql, update_version_stamp, show_progress +) + +installdir = get_install_dir() +topdir = dirname(installdir) + +def collect_upgrade_scripts(from_version, to_version): + """ + Give the current installed version, calculate which upgrade scripts we need + to run to upgrade it to the latest verison. + + For example, given current version 5.0.1 and target version 6.1.0, and these + upgrade scripts: + + upgrade_4.4_5.0.sh + upgrade_5.0_5.1.sh + upgrade_5.1_6.0.sh + upgrade_6.0_6.1.sh + + We need to run upgrade_5.0_5.1.sh, upgrade_5.1_6.0.sh, and upgrade_6.0_6.1.sh. + """ + from_major_ver = '.'.join(from_version.split('.')[:2]) + to_major_ver = '.'.join(to_version.split('.')[:2]) + + scripts = [] + for fn in glob.glob(join(installdir, 'upgrade', 'upgrade_*_*.sh')): + va, vb = parse_upgrade_script_version(fn) + if va >= from_major_ver and vb <= to_major_ver: + scripts.append(fn) + return scripts + +def parse_upgrade_script_version(script): + script = basename(script) + m = re.match(r'upgrade_([0-9+.]+)_([0-9+.]+).sh', basename(script)) + return m.groups() + +def check_upgrade(): + last_version = read_version_stamp() + current_version = os.environ['SEAFILE_VERSION'] + if last_version == current_version: + return + + scripts_to_run = collect_upgrade_scripts(from_version=last_version, to_version=current_version) + for script in scripts_to_run: + show_progress('Running scripts {}'.format(script)) + new_version = parse_upgrade_script_version(script)[1] + '.0' + + replace_file_pattern(script, 'read dummy', '') + call(script) + + update_version_stamp(new_version) + + update_version_stamp(current_version) + +def main(): + wait_for_mysql() + + os.chdir(installdir) + check_upgrade() + +if __name__ == '__main__': + main() diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py index fc4eab8..9c53408 100644 --- a/scripts/utils/__init__.py +++ b/scripts/utils/__init__.py @@ -9,6 +9,7 @@ from os.path import abspath, basename, exists, dirname, join, isdir, expanduser import platform import sys import subprocess +import time import logging import logging.config import click @@ -251,11 +252,23 @@ def cert_has_valid_days(cert, days): def get_version_stamp_file(): return '/shared/seafile/seafile-data/current_version' -def read_version_stamp(fn): +def read_version_stamp(fn=get_version_stamp_file()): assert exists(fn), 'version stamp file {} does not exist!'.format(fn) with open(fn, 'r') as fp: return fp.read().strip() -def update_version_stamp(fn, version): +def update_version_stamp(version, fn=get_version_stamp_file()): with open(fn, 'w') as fp: fp.write(version + '\n') + +def wait_for_mysql(): + while not exists('/var/run/mysqld/mysqld.sock'): + print('waiting for mysql server to be ready') + time.sleep(2) + print('mysql server is ready') + +def replace_file_pattern(fn, pattern, replacement): + with open(fn, 'r') as fp: + content = fp.read() + with open(fn, 'w') as fp: + fp.write(content.replace(pattern, replacement)) diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..d386bb6 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,7 @@ +FROM seafileorg/server:6.0.5 + +ENV SEAFILE_VERSION=6.1.0 + +RUN mv /opt/seafile/seafile-server-6.0.5 /opt/seafile/seafile-server-${SEAFILE_VERSION} + +ADD upgrade_6.0_6.1.sh /opt/seafile/seafile-server-${SEAFILE_VERSION}/upgrade/upgrade_6.0_6.1.sh diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..685075f --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,2 @@ +all: + docker build -t seafileorg/server:6.1.0 . diff --git a/tests/upgrade_6.0_6.1.sh b/tests/upgrade_6.0_6.1.sh new file mode 100755 index 0000000..e858212 --- /dev/null +++ b/tests/upgrade_6.0_6.1.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +SCRIPT=$(readlink -f "$0") # haiwen/seafile-server-1.3.0/upgrade/upgrade_xx_xx.sh +UPGRADE_DIR=$(dirname "$SCRIPT") # haiwen/seafile-server-1.3.0/upgrade/ +INSTALLPATH=$(dirname "$UPGRADE_DIR") # haiwen/seafile-server-1.3.0/ +TOPDIR=$(dirname "${INSTALLPATH}") # haiwen/ +default_ccnet_conf_dir=${TOPDIR}/ccnet +default_conf_dir=${TOPDIR}/conf +seafile_server_symlink=${TOPDIR}/seafile-server-latest +seahub_data_dir=${TOPDIR}/seahub-data +seahub_settings_py=${TOPDIR}/seahub_settings.py + +manage_py=${INSTALLPATH}/seahub/manage.py + +export CCNET_CONF_DIR=${default_ccnet_conf_dir} +export SEAFILE_CENTRAL_CONF_DIR=${default_conf_dir} +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.6/site-packages:${INSTALLPATH}/seafile/lib64/python2.6/site-packages:${INSTALLPATH}/seafile/lib/python2.7/site-packages:${INSTALLPATH}/seahub/thirdpart:$PYTHONPATH +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.7/site-packages:${INSTALLPATH}/seafile/lib64/python2.7/site-packages:$PYTHONPATH +export SEAFILE_LD_LIBRARY_PATH=${INSTALLPATH}/seafile/lib/:${INSTALLPATH}/seafile/lib64:${LD_LIBRARY_PATH} + +prev_version=6.0 +current_version=6.1 + +echo +echo "-------------------------------------------------------------" +echo "This script would upgrade your seafile server from ${prev_version} to ${current_version}" +echo "Press [ENTER] to contiune" +echo "-------------------------------------------------------------" +echo +read dummy + +function check_python_executable() { + if [[ "$PYTHON" != "" && -x $PYTHON ]]; then + return 0 + fi + + if which python2.7 2>/dev/null 1>&2; then + PYTHON=python2.7 + elif which python27 2>/dev/null 1>&2; then + PYTHON=python27 + else + echo + echo "Can't find a python executable of version 2.7 or above in PATH" + echo "Install python 2.7+ before continue." + echo "Or if you installed it in a non-standard PATH, set the PYTHON enviroment varirable to it" + echo + exit 1 + fi +} + +function read_seafile_data_dir () { + seafile_ini=${default_ccnet_conf_dir}/seafile.ini + if [[ ! -f ${seafile_ini} ]]; then + echo "${seafile_ini} not found. Now quit" + exit 1 + fi + seafile_data_dir=$(cat "${seafile_ini}") + if [[ ! -d ${seafile_data_dir} ]]; then + echo "Your seafile server data directory \"${seafile_data_dir}\" is invalid or doesn't exits." + echo "Please check it first, or create this directory yourself." + echo "" + exit 1; + fi + + export SEAFILE_CONF_DIR=$seafile_data_dir +} + +function ensure_server_not_running() { + # test whether seafile server has been stopped. + if pgrep seaf-server 2>/dev/null 1>&2 ; then + echo + echo "seafile server is still running !" + echo "stop it using scripts before upgrade." + echo + exit 1 + elif pgrep -f "${manage_py} run_gunicorn" 2>/dev/null 1>&2 \ + || pgrep -f "seahub.wsgi:application" 2>/dev/null 1>&2; then + echo + echo "seahub server is still running !" + echo "stop it before upgrade." + echo + exit 1 + elif pgrep -f "${manage_py} runfcgi" 2>/dev/null 1>&2 ; then + echo + echo "seahub server is still running !" + echo "stop it before upgrade." + echo + exit 1 + fi +} + +function migrate_avatars() { + echo + echo "migrating avatars ..." + echo + media_dir=${INSTALLPATH}/seahub/media + orig_avatar_dir=${INSTALLPATH}/seahub/media/avatars + dest_avatar_dir=${TOPDIR}/seahub-data/avatars + + # move "media/avatars" directory outside + if [[ ! -d ${dest_avatar_dir} ]]; then + mkdir -p "${TOPDIR}/seahub-data" + mv "${orig_avatar_dir}" "${dest_avatar_dir}" 2>/dev/null 1>&2 + ln -s ../../../seahub-data/avatars "${media_dir}" + + elif [[ ! -L ${orig_avatar_dir} ]]; then + mv "${orig_avatar_dir}"/* "${dest_avatar_dir}" 2>/dev/null 1>&2 + rm -rf "${orig_avatar_dir}" + ln -s ../../../seahub-data/avatars "${media_dir}" + fi + echo "Done" +} + +function update_database() { + echo + echo "Updating seafile/seahub database ..." + echo + + db_update_helper=${UPGRADE_DIR}/db_update_helper.py + if ! $PYTHON "${db_update_helper}" 6.0.0; then + echo + echo "Failed to upgrade your database" + echo + exit 1 + fi + echo "Done" +} + +function upgrade_seafile_server_latest_symlink() { + # update the symlink seafile-server to the new server version + if [[ -L "${seafile_server_symlink}" || ! -e "${seafile_server_symlink}" ]]; then + echo + printf "updating \033[33m${seafile_server_symlink}\033[m symbolic link to \033[33m${INSTALLPATH}\033[m ...\n\n" + echo + if ! rm -f "${seafile_server_symlink}"; then + echo "Failed to remove ${seafile_server_symlink}" + echo + exit 1; + fi + + if ! ln -s "$(basename ${INSTALLPATH})" "${seafile_server_symlink}"; then + echo "Failed to update ${seafile_server_symlink} symbolic link." + echo + exit 1; + fi + fi +} + +function make_media_custom_symlink() { + media_symlink=${INSTALLPATH}/seahub/media/custom + if [[ -L "${media_symlink}" ]]; then + return + + elif [[ ! -e "${media_symlink}" ]]; then + ln -s ../../../seahub-data/custom "${media_symlink}" + return + + + elif [[ -d "${media_symlink}" ]]; then + cp -rf "${media_symlink}" "${seahub_data_dir}/" + rm -rf "${media_symlink}" + ln -s ../../../seahub-data/custom "${media_symlink}" + fi + +} + +function move_old_customdir_outside() { + # find the path of the latest seafile server folder + if [[ -L ${seafile_server_symlink} ]]; then + latest_server=$(readlink -f "${seafile_server_symlink}") + else + return + fi + + old_customdir=${latest_server}/seahub/media/custom + + # old customdir is already a symlink, do nothing + if [[ -L "${old_customdir}" ]]; then + return + fi + + # old customdir does not exist, do nothing + if [[ ! -e "${old_customdir}" ]]; then + return + fi + + # media/custom exist and is not a symlink + cp -rf "${old_customdir}" "${seahub_data_dir}/" +} + +################# +# The main execution flow of the script +################ + +check_python_executable; +read_seafile_data_dir; +ensure_server_not_running; + +# update_database; +migrate_avatars; + +move_old_customdir_outside; +make_media_custom_symlink; +upgrade_seafile_server_latest_symlink; + +echo +echo "-----------------------------------------------------------------" +echo "Upgraded your seafile server successfully." +echo "-----------------------------------------------------------------" +echo