diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000..ee4424b --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,17 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 90 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 7 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security +# Label to use when marking an issue as stale +staleLabel: wontfix +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false diff --git a/README.pro.md b/README.pro.md index 28c8e27..4bda6e2 100644 --- a/README.pro.md +++ b/README.pro.md @@ -34,6 +34,21 @@ Wait for a few minutes for the first time initialization, then visit `http://sea This command will mount folder `/opt/seafile-data` at the local server to the docker instance. You can find logs and other data under this folder. +### Put your licence file + +If you have a `seafile-license.txt` licence file, simply put it in the folder `/opt/seafile-data/seafile/`. In your host machine: + +```sh +mkdir -p /opt/seafile-data/seafile/ +cp /path/to/seafile-license.txt /opt/seafile-data/seafile/ +``` + +Then restart the container. + +```sh +docker restart seafile +``` + ### More configuration Options #### Custom Admin Username and Password diff --git a/README.windows.md b/README.windows.md index d98ceab..b9502eb 100644 --- a/README.windows.md +++ b/README.windows.md @@ -55,8 +55,10 @@ Now run the following commands: (Note that if you're using another drive than "C:", say "D:", you should change the "c:\\seafile" in the following commands to "d:\\seafile" instead.) ```sh -docker pull seafileltd/seafile:6.2.1 -docker run -d --name seafile-server -v /root/seafile:/shared -p 80:80 seafileltd/seafile:6.2.1 +docker pull seafileltd/seafile:6.3.3 +docker run -d --name seafile-server -v /root/seafile:/shared -p 80:80 seafileltd/seafile:6.3.3 ``` +The tag for the most recent version of the image can be found at https://hub.docker.com/r/seafileltd/seafile/tags/. + If you are not familiar with docker commands, refer to [docker documentation](https://docs.docker.com/engine/reference/commandline/cli/). diff --git a/cluster/image/Makefile b/cluster/image/Makefile index 8167007..8f68a65 100644 --- a/cluster/image/Makefile +++ b/cluster/image/Makefile @@ -1,14 +1,14 @@ -server_version=6.2.13 +server_version=6.3.7 -base_image=seafileltd/base:16.04 -base_image_squashed=seafileltd/base:16.04-squashed -pro_base_image=seafileltd/pro-base:16.04 -pro_base_image_squashed=seafileltd/pro-base:16.04-squashed +base_image=seafileltd/cluster-base:18.04 +base_image_squashed=seafileltd/cluster-base:18.04-squashed +pro_base_image=seafileltd/cluster-pro-base:18.04 +pro_base_image_squashed=seafileltd/cluster-pro-base:18.04-squashed server_image=seafileltd/seafile:$(server_version) server_image_squashed=seafileltd/seafile:$(server_version)-squashed -pro_server_image=seafileltd/seafile-pro:$(server_version) -pro_server_image_squashed=seafileltd/seafile-pro:$(server_version)-squashed -latest_pro_server_image=seafileltd/seafile-pro:latest +pro_server_image=seafileltd/cluster-seafile-pro:$(server_version) +pro_server_image_squashed=seafileltd/cluster-seafile-pro:$(server_version)-squashed +latest_pro_server_image=seafileltd/cluster-seafile-pro:latest latest_server_image=seafileltd/seafile:latest all: @@ -17,9 +17,9 @@ all: @echo base: - docker pull phusion/baseimage:0.9.19 - docker-squash --tag phusion/baseimage:latest phusion/baseimage:0.9.19 - docker tag phusion/baseimage:latest phusion/baseimage:0.9.19 + docker pull phusion/baseimage:0.11 + docker-squash --tag phusion/baseimage:latest phusion/baseimage:0.11 + docker tag phusion/baseimage:latest phusion/baseimage:0.11 cd base && docker build -t $(base_image) . docker-squash --tag $(base_image_squashed) $(base_image) docker tag $(base_image_squashed) $(base_image) diff --git a/cluster/image/base/Dockerfile b/cluster/image/base/Dockerfile index 9d9baa0..d4897de 100644 --- a/cluster/image/base/Dockerfile +++ b/cluster/image/base/Dockerfile @@ -1,6 +1,6 @@ -# Lastet phusion baseimage as of 20180412, based on ubuntu 16.04 +# Lastet phusion baseimage as of 20180412, based on ubuntu 18.04 # See https://hub.docker.com/r/phusion/baseimage/tags/ -FROM phusion/baseimage:0.10.1 +FROM phusion/baseimage:0.11 ENV UPDATED_AT=20180412 \ DEBIAN_FRONTEND=noninteractive @@ -15,7 +15,7 @@ RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl # Guidline for installing python libs: if a lib has C-compoment (e.g. # python-imaging depends on libjpeg/libpng), we install it use apt-get. # Otherwise we install it with pip. -RUN apt-get install -y python2.7-dev python-imaging python-ldap python-mysqldb +RUN apt-get install -y python2.7-dev python-ldap python-mysqldb libmemcached-dev zlib1g-dev gcc RUN curl -sSL -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ python /tmp/get-pip.py && \ rm -rf /tmp/get-pip.py && \ diff --git a/cluster/image/base/requirements.txt b/cluster/image/base/requirements.txt index 3094155..e107f7b 100644 --- a/cluster/image/base/requirements.txt +++ b/cluster/image/base/requirements.txt @@ -9,7 +9,8 @@ colorlog==2.7.0 Jinja2==2.8 MarkupSafe==0.23 # via jinja2 prettytable==0.7.2 -python-memcached==1.58 -six==1.10.0 # via python-memcached termcolor==1.1.0 urllib3==1.19 +Pillow==4.3.0 +pylibmc +django-pylibmc diff --git a/cluster/image/pro_base/Dockerfile b/cluster/image/pro_base/Dockerfile index 7f095f1..cbbe8d2 100644 --- a/cluster/image/pro_base/Dockerfile +++ b/cluster/image/pro_base/Dockerfile @@ -1,15 +1,25 @@ -FROM seafileltd/base:16.04 +FROM seafileltd/cluster-base:18.04 # syslog-ng and syslog-forwarder would mess up the container stdout, not good # when debugging/upgrading. + +# Fixing the "Sub-process /usr/bin/dpkg returned an error code (1)", +# when RUN apt-get +RUN mkdir -p /usr/share/man/man1 + RUN apt update RUN apt-get install -y openjdk-8-jre libmemcached-dev zlib1g-dev pwgen curl openssl poppler-utils libpython2.7 libreoffice \ -libreoffice-script-provider-python ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy python-requests +libreoffice-script-provider-python ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy python-requests mysql-client RUN apt-get install -y tzdata python-pip python-setuptools python-urllib3 python-ldap python-ceph -RUN pip install pylibmc django-pylibmc boto twilio oss2 - +# The S3 storage, oss storage and psd online preview etc, +# depends on the python-backages as follow: +RUN pip install boto==2.43.0 \ + oss2==2.3.0 \ + psd-tools==1.4 \ + pycryptodome==3.7.2 \ + twilio==5.7.0 RUN apt clean diff --git a/cluster/image/pro_seafile/Dockerfile b/cluster/image/pro_seafile/Dockerfile index 1927997..fbc4c9b 100644 --- a/cluster/image/pro_seafile/Dockerfile +++ b/cluster/image/pro_seafile/Dockerfile @@ -1,13 +1,13 @@ -FROM seafileltd/pro-base:16.04 +FROM seafileltd/cluster-pro-base:18.04 WORKDIR /opt/seafile -ENV SEAFILE_VERSION=6.2.13 SEAFILE_SERVER=seafile-pro-server +ENV SEAFILE_VERSION=6.3.7 SEAFILE_SERVER=seafile-pro-server RUN mkdir -p /etc/my_init.d RUN mkdir -p /opt/seafile/ -RUN curl -sSL -G -d "p=/seafile-pro-server_${SEAFILE_VERSION}_x86-64_Ubuntu.tar.gz&dl=1" https://download.seafile.top/d/8c29766a64d24122936f/files/ \ +RUN curl -sSL -G -d "p=/pro/seafile-pro-server_${SEAFILE_VERSION}_x86-64_Ubuntu.tar.gz&dl=1" https://download.seafile.com/d/6e5297246c/files/ \ | tar xzf - -C /opt/seafile/ ADD scripts/create_data_links.sh /etc/my_init.d/01_create_data_links.sh diff --git a/cluster/image/pro_seafile/scripts/bootstrap.py b/cluster/image/pro_seafile/scripts/bootstrap.py new file mode 100755 index 0000000..bfba2aa --- /dev/null +++ b/cluster/image/pro_seafile/scripts/bootstrap.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +Bootstraping seafile server, letsencrypt (verification & cron job). +""" + +import argparse +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys +import uuid +import time + +from utils import ( + call, get_conf, get_install_dir, loginfo, + get_script, render_template, get_seafile_version, eprint, + cert_has_valid_days, get_version_stamp_file, update_version_stamp, + wait_for_mysql, wait_for_nginx, read_version_stamp +) + +seafile_version = get_seafile_version() +installdir = get_install_dir() +topdir = dirname(installdir) +shared_seafiledir = '/shared/seafile' +ssl_dir = '/shared/ssl' +generated_dir = '/bootstrap/generated' + +def init_letsencrypt(): + loginfo('Preparing for letsencrypt ...') + wait_for_nginx() + + if not exists(ssl_dir): + os.mkdir(ssl_dir) + + domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com') + context = { + 'ssl_dir': ssl_dir, + 'domain': domain, + } + render_template( + '/templates/letsencrypt.cron.template', + join(generated_dir, 'letsencrypt.cron'), + context + ) + + ssl_crt = '/shared/ssl/{}.crt'.format(domain) + if exists(ssl_crt): + loginfo('Found existing cert file {}'.format(ssl_crt)) + if cert_has_valid_days(ssl_crt, 30): + loginfo('Skip letsencrypt verification since we have a valid certificate') + return + + loginfo('Starting letsencrypt verification') + # Create a temporary nginx conf to start a server, which would accessed by letsencrypt + context = { + 'https': False, + 'domain': domain, + } + render_template('/templates/seafile.nginx.conf.template', + '/etc/nginx/sites-enabled/seafile.nginx.conf', context) + + call('nginx -s reload') + time.sleep(2) + + call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain)) + # if call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain), check_call=False) != 0: + # eprint('Now waiting 1000s for postmortem') + # time.sleep(1000) + # sys.exit(1) + + +def generate_local_nginx_conf(): + # Now create the final nginx configuratin + domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com') + context = { + 'https': is_https(), + 'domain': domain, + } + render_template( + '/templates/seafile.nginx.conf.template', + '/etc/nginx/sites-enabled/seafile.nginx.conf', + context + ) + + +def is_https(): + return get_conf('SEAFILE_SERVER_LETSENCRYPT', 'false').lower() == 'true' + +def parse_args(): + ap = argparse.ArgumentParser() + ap.add_argument('--parse-ports', action='store_true') + + return ap.parse_args() + +def init_seafile_server(): + version_stamp_file = get_version_stamp_file() + if exists(join(shared_seafiledir, 'seafile-data')): + if not exists(version_stamp_file): + update_version_stamp(os.environ['SEAFILE_VERSION']) + # sysbol link unlink after docker finish. + latest_version_dir='/opt/seafile/seafile-server-latest' + current_version_dir='/opt/seafile/' + get_conf('SEAFILE_SERVER', 'seafile-server') + '-' + read_version_stamp() + if not exists(latest_version_dir): + call('ln -sf ' + current_version_dir + ' ' + latest_version_dir) + loginfo('Skip running setup-seafile-mysql.py because there is existing seafile-data folder.') + return + + loginfo('Now running setup-seafile-mysql.py in auto mode.') + env = { + 'SERVER_NAME': 'seafile', + 'SERVER_IP': get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com'), + 'MYSQL_USER': 'seafile', + 'MYSQL_USER_PASSWD': str(uuid.uuid4()), + 'MYSQL_USER_HOST': '127.0.0.1', + # Default MariaDB root user has empty password and can only connect from localhost. + 'MYSQL_ROOT_PASSWD': '', + } + + # Change the script to allow mysql root password to be empty + call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}''' + .format(get_script('setup-seafile-mysql.py'))) + + setup_script = get_script('setup-seafile-mysql.sh') + call('{} auto -n seafile'.format(setup_script), env=env) + + domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com') + proto = 'https' if is_https() else 'http' + with open(join(topdir, 'conf', 'seahub_settings.py'), 'a+') as fp: + fp.write('\n') + fp.write('FILE_SERVER_ROOT = "{proto}://{domain}/seafhttp"'.format(proto=proto, domain=domain)) + fp.write('\n') + + # By default ccnet-server binds to the unix socket file + # "/opt/seafile/ccnet/ccnet.sock", but /opt/seafile/ccnet/ is a mounted + # volume from the docker host, and on windows and some linux environment + # it's not possible to create unix sockets in an external-mounted + # directories. So we change the unix socket file path to + # "/opt/seafile/ccnet.sock" to avoid this problem. + with open(join(topdir, 'conf', 'ccnet.conf'), 'a+') as fp: + fp.write('\n') + fp.write('[Client]\n') + fp.write('UNIX_SOCKET = /opt/seafile/ccnet.sock\n') + fp.write('\n') + + files_to_copy = ['conf', 'ccnet', 'seafile-data', 'seahub-data', 'pro-data'] + for fn in files_to_copy: + src = join(topdir, fn) + dst = join(shared_seafiledir, fn) + if not exists(dst) and exists(src): + shutil.move(src, shared_seafiledir) + call('ln -sf ' + join(shared_seafiledir, fn) + ' ' + src) + + loginfo('Updating version stamp') + update_version_stamp(os.environ['SEAFILE_VERSION']) diff --git a/cluster/image/pro_seafile/scripts/create_data_links.sh b/cluster/image/pro_seafile/scripts/create_data_links.sh new file mode 100755 index 0000000..db91ff9 --- /dev/null +++ b/cluster/image/pro_seafile/scripts/create_data_links.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [[ $SEAFILE_BOOTSRAP != "" ]]; then + exit 0 +fi + +if [[ $TIME_ZONE != "" ]]; then + time_zone=/usr/share/zoneinfo/$TIME_ZONE + if [[ ! -e $time_zone ]]; then + echo "invalid time zone" + exit 1 + else + ln -snf $time_zone /etc/localtime + echo "$TIME_ZONE" > /etc/timezone + fi +fi + +dirs=( + conf + ccnet + seafile-data + seahub-data + pro-data + seafile-license.txt +) + +for d in ${dirs[*]}; do + src=/shared/seafile/$d + if [[ -e $src ]]; then + rm -rf /opt/seafile/$d && ln -sf $src /opt/seafile + fi +done + +if [[ ! -e /shared/logs/seafile ]]; then + mkdir -p /shared/logs/seafile +fi +rm -rf /opt/seafile/logs && ln -sf /shared/logs/seafile/ /opt/seafile/logs + +current_version_dir=/opt/seafile/${SEAFILE_SERVER}-${SEAFILE_VERSION} +latest_version_dir=/opt/seafile/seafile-server-latest +seahub_data_dir=/shared/seafile/seahub-data + +if [[ ! -e $seahub_data_dir ]]; then + mkdir -p $seahub_data_dir +fi + +media_dirs=( + avatars + custom +) +for d in ${media_dirs[*]}; do + source_media_dir=${current_version_dir}/seahub/media/$d + if [ -e ${source_media_dir} ] && [ ! -e ${seahub_data_dir}/$d ]; then + mv $source_media_dir ${seahub_data_dir}/$d + fi + rm -rf $source_media_dir && ln -sf ${seahub_data_dir}/$d $source_media_dir +done + +rm -rf /var/lib/mysql +if [[ ! -e /shared/db ]];then + mkdir -p /shared/db +fi +ln -sf /shared/db /var/lib/mysql + +if [[ ! -e /shared/logs/var-log ]]; then + chmod 777 /var/log -R + mv /var/log /shared/logs/var-log +fi +rm -rf /var/log && ln -sf /shared/logs/var-log /var/log + +if [[ ! -e latest_version_dir ]]; then + ln -sf $current_version_dir $latest_version_dir +fi + +chmod u+x /scripts/* + +echo $PYTHON +$PYTHON /scripts/init.py diff --git a/cluster/image/pro_seafile/scripts/gc.sh b/cluster/image/pro_seafile/scripts/gc.sh new file mode 100755 index 0000000..4531933 --- /dev/null +++ b/cluster/image/pro_seafile/scripts/gc.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +# Before +SEAFILE_DIR=/opt/seafile/seafile-server-latest + +if [[ $SEAFILE_SERVER != *"pro"* ]]; then + echo "Seafile CE: Stop Seafile to perform offline garbage collection." + $SEAFILE_DIR/seafile.sh stop + + echo "Waiting for the server to shut down properly..." + sleep 5 +else + echo "Seafile Pro: Perform online garbage collection." +fi + +# Do it +( + set +e + $SEAFILE_DIR/seaf-gc.sh "$@" | tee -a /var/log/gc.log + # We want to presevent the exit code of seaf-gc.sh + exit "${PIPESTATUS[0]}" +) + +gc_exit_code=$? + +# After + +if [[ $SEAFILE_SERVER != *"pro"* ]]; then + echo "Giving the server some time..." + sleep 3 + + $SEAFILE_DIR/seafile.sh start +fi + +exit $gc_exit_code diff --git a/cluster/image/pro_seafile/scripts/init.py b/cluster/image/pro_seafile/scripts/init.py new file mode 100755 index 0000000..6d3b80b --- /dev/null +++ b/cluster/image/pro_seafile/scripts/init.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +Starts the seafile/seahub server and watches the controller process. It is +the entrypoint command of the docker container. +""" + +import json +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys +import time + +from utils import ( + call, get_conf, get_install_dir, get_script, get_command_output, + render_template, wait_for_mysql +) +from upgrade import check_upgrade +from bootstrap import init_seafile_server, is_https, init_letsencrypt, generate_local_nginx_conf + + +shared_seafiledir = '/shared/seafile' +ssl_dir = '/shared/ssl' +generated_dir = '/bootstrap/generated' +installdir = get_install_dir() +topdir = dirname(installdir) + + +def main(): + call('cp -rf /scripts/setup-seafile-mysql.py ' + join(installdir, 'setup-seafile-mysql.py')) + if not exists(shared_seafiledir): + os.mkdir(shared_seafiledir) + if not exists(generated_dir): + os.makedirs(generated_dir) + + if is_https(): + init_letsencrypt() + generate_local_nginx_conf() + + if not exists(join(shared_seafiledir, 'conf')): + init_seafile_server() + +if __name__ == '__main__': + main() diff --git a/cluster/image/pro_seafile/scripts/setup-seafile-mysql.py b/cluster/image/pro_seafile/scripts/setup-seafile-mysql.py new file mode 100755 index 0000000..dd71cea --- /dev/null +++ b/cluster/image/pro_seafile/scripts/setup-seafile-mysql.py @@ -0,0 +1,1497 @@ +#coding: UTF-8 + +'''This script would guide the seafile admin to setup seafile with MySQL''' + +import argparse +import sys +import os +import time +import re +import shutil +import glob +import subprocess +import hashlib +import getpass +import uuid +import warnings +import socket +from ConfigParser import ConfigParser + +import MySQLdb + + +try: + import readline # pylint: disable=W0611 +except ImportError: + pass + + +SERVER_MANUAL_HTTP = 'https://github.com/haiwen/seafile/wiki' + +class Utils(object): + '''Groups all helper functions here''' + @staticmethod + def welcome(): + '''Show welcome message''' + welcome_msg = '''\ +----------------------------------------------------------------- +This script will guide you to setup your seafile server using MySQL. +Make sure you have read seafile server manual at + + %s + +Press ENTER to continue +-----------------------------------------------------------------''' % SERVER_MANUAL_HTTP + print welcome_msg + raw_input() + + @staticmethod + def highlight(content): + '''Add ANSI color to content to get it highlighted on terminal''' + return '\x1b[33m%s\x1b[m' % content + + @staticmethod + def info(msg): + print msg + + @staticmethod + def error(msg): + '''Print error and exit''' + print + print 'Error: ' + msg + sys.exit(1) + + @staticmethod + def run_argv(argv, cwd=None, env=None, suppress_stdout=False, suppress_stderr=False): + '''Run a program and wait it to finish, and return its exit code. The + standard output of this program is supressed. + + ''' + with open(os.devnull, 'w') as devnull: + if suppress_stdout: + stdout = devnull + else: + stdout = sys.stdout + + if suppress_stderr: + stderr = devnull + else: + stderr = sys.stderr + + proc = subprocess.Popen(argv, + cwd=cwd, + stdout=stdout, + stderr=stderr, + env=env) + return proc.wait() + + @staticmethod + def run(cmdline, cwd=None, env=None, suppress_stdout=False, suppress_stderr=False): + '''Like run_argv but specify a command line string instead of argv''' + with open(os.devnull, 'w') as devnull: + if suppress_stdout: + stdout = devnull + else: + stdout = sys.stdout + + if suppress_stderr: + stderr = devnull + else: + stderr = sys.stderr + + proc = subprocess.Popen(cmdline, + cwd=cwd, + stdout=stdout, + stderr=stderr, + env=env, + shell=True) + return proc.wait() + + @staticmethod + def get_command_output(args, *a, **kw): + return subprocess.check_output(args, *a, **kw) + + @staticmethod + def prepend_env_value(name, value, env=None, seperator=':'): + '''prepend a new value to a list''' + if env is None: + env = os.environ + + try: + current_value = env[name] + except KeyError: + current_value = '' + + new_value = value + if current_value: + new_value += seperator + current_value + + env[name] = new_value + + @staticmethod + def must_mkdir(path): + '''Create a directory, exit on failure''' + if os.path.exists(path): + return + try: + os.mkdir(path) + except OSError, e: + Utils.error('failed to create directory %s:%s' % (path, e)) + + @staticmethod + def must_copy(src, dst): + '''Copy src to dst, exit on failure''' + try: + shutil.copy(src, dst) + except Exception, e: + Utils.error('failed to copy %s to %s: %s' % (src, dst, e)) + + @staticmethod + def find_in_path(prog): + if 'win32' in sys.platform: + sep = ';' + else: + sep = ':' + + dirs = os.environ['PATH'].split(sep) + for d in dirs: + d = d.strip() + if d == '': + continue + path = os.path.join(d, prog) + if os.path.exists(path): + return path + + return None + + @staticmethod + def get_python_executable(): + '''Return the python executable. This should be the PYTHON environment + variable which is set in setup-seafile-mysql.sh + + ''' + return os.environ['PYTHON'] + + @staticmethod + def read_config(fn): + '''Return a case sensitive ConfigParser by reading the file "fn"''' + cp = ConfigParser() + cp.optionxform = str + cp.read(fn) + + return cp + + @staticmethod + def write_config(cp, fn): + '''Return a case sensitive ConfigParser by reading the file "fn"''' + with open(fn, 'w') as fp: + cp.write(fp) + + @staticmethod + def ask_question(desc, + key=None, + note=None, + default=None, + validate=None, + yes_or_no=False, + password=False): + '''Ask a question, return the answer. + @desc description, e.g. "What is the port of ccnet?" + + @key a name to represent the target of the question, e.g. "port for + ccnet server" + + @note additional information for the question, e.g. "Must be a valid + port number" + + @default the default value of the question. If the default value is + not None, when the user enter nothing and press [ENTER], the default + value would be returned + + @validate a function that takes the user input as the only parameter + and validate it. It should return a validated value, or throws an + "InvalidAnswer" exception if the input is not valid. + + @yes_or_no If true, the user must answer "yes" or "no", and a boolean + value would be returned + + @password If true, the user input would not be echoed to the + console + + ''' + assert key or yes_or_no + # Format description + print + if note: + desc += '\n' + note + + desc += '\n' + if yes_or_no: + desc += '[ yes or no ]' + else: + if default: + desc += '[ default "%s" ]' % default + else: + desc += '[ %s ]' % key + + desc += ' ' + while True: + # prompt for user input + if password: + answer = getpass.getpass(desc).strip() + else: + answer = raw_input(desc).strip() + + # No user input: use default + if not answer: + if default: + answer = default + else: + continue + + # Have user input: validate answer + if yes_or_no: + if answer not in ['yes', 'no']: + print Utils.highlight('\nPlease answer yes or no\n') + continue + else: + return answer == 'yes' + else: + if validate: + try: + return validate(answer) + except InvalidAnswer, e: + print Utils.highlight('\n%s\n' % e) + continue + else: + return answer + + @staticmethod + def validate_port(port): + try: + port = int(port) + except ValueError: + raise InvalidAnswer('%s is not a valid port' % Utils.highlight(port)) + + if port <= 0 or port > 65535: + raise InvalidAnswer('%s is not a valid port' % Utils.highlight(port)) + + return port + + +class InvalidAnswer(Exception): + def __init__(self, msg): + Exception.__init__(self) + self.msg = msg + def __str__(self): + return self.msg + +class InvalidParams(Exception): + def __init__(self, msg): + Exception.__init__(self) + self.msg = msg + def __str__(self): + return self.msg + +### END of Utils +#################### + +class EnvManager(object): + '''System environment and directory layout''' + def __init__(self): + self.install_path = os.path.dirname(os.path.abspath(__file__)) + self.top_dir = os.path.dirname(self.install_path) + self.bin_dir = os.path.join(self.install_path, 'seafile', 'bin') + self.central_config_dir = os.path.join(self.top_dir, 'conf') + Utils.must_mkdir(self.central_config_dir) + + def check_pre_condiction(self): + def error_if_not_exists(path): + if not os.path.exists(path): + Utils.error('"%s" not found' % path) + + paths = [ + os.path.join(self.install_path, 'seafile'), + os.path.join(self.install_path, 'seahub'), + os.path.join(self.install_path, 'runtime'), + ] + + for path in paths: + error_if_not_exists(path) + + if os.path.exists(ccnet_config.ccnet_dir): + Utils.error('Ccnet config dir \"%s\" already exists.' % ccnet_config.ccnet_dir) + + def get_seahub_env(self): + '''Prepare for seahub syncdb''' + env = dict(os.environ) + env['CCNET_CONF_DIR'] = ccnet_config.ccnet_dir + env['SEAFILE_CONF_DIR'] = seafile_config.seafile_dir + env['SEAFES_DIR'] = os.path.join(self.install_path, 'pro', 'python', 'seafes') + self.setup_python_path(env) + return env + + def setup_python_path(self, env): + '''And PYTHONPATH and CCNET_CONF_DIR/SEAFILE_CONF_DIR to env, which is + needed by seahub + + ''' + install_path = self.install_path + pro_pylibs_dir = os.path.join(install_path, 'pro', 'python') + extra_python_path = [ + pro_pylibs_dir, + + os.path.join(install_path, 'seahub', 'thirdpart'), + os.path.join(install_path, 'seahub-extra'), + os.path.join(install_path, 'seahub-extra', 'thirdparts'), + + os.path.join(install_path, 'seafile/lib/python2.6/site-packages'), + os.path.join(install_path, 'seafile/lib64/python2.6/site-packages'), + os.path.join(install_path, 'seafile/lib/python2.7/site-packages'), + os.path.join(install_path, 'seafile/lib64/python2.7/site-packages'), + ] + + for path in extra_python_path: + Utils.prepend_env_value('PYTHONPATH', path, env=env) + + def get_binary_env(self): + '''Set LD_LIBRARY_PATH for seafile server executables''' + env = dict(os.environ) + lib_dir = os.path.join(self.install_path, 'seafile', 'lib') + lib64_dir = os.path.join(self.install_path, 'seafile', 'lib64') + Utils.prepend_env_value('LD_LIBRARY_PATH', lib_dir, env=env) + Utils.prepend_env_value('LD_LIBRARY_PATH', lib64_dir, env=env) + return env + +class AbstractConfigurator(object): + '''Abstract Base class for ccnet/seafile/seahub/db configurator''' + def __init__(self): + pass + + def ask_questions(self): + raise NotImplementedError + + def generate(self): + raise NotImplementedError + + +class AbstractDBConfigurator(AbstractConfigurator): + '''Abstract class for database related configuration''' + def __init__(self): + AbstractConfigurator.__init__(self) + self.mysql_host = 'localhost' + self.mysql_port = 3306 + + self.use_existing_db = False + + self.seafile_mysql_user = '' + self.seafile_mysql_password = '' + self.seafile_mysql_userhost = 'localhost' + + self.root_password = '' + self.root_conn = '' + + self.ccnet_db_name = '' + self.seafile_db_name = '' + self.seahub_db_name = '' + + self.seahub_admin_email = '' + self.seahub_admin_password = '' + + @staticmethod + def ask_use_existing_db(): + def validate(choice): + if choice not in ['1', '2']: + raise InvalidAnswer('Please choose 1 or 2') + + return choice == '2' + + question = '''\ +------------------------------------------------------- +Please choose a way to initialize seafile databases: +------------------------------------------------------- +''' + + note = '''\ +[1] Create new ccnet/seafile/seahub databases +[2] Use existing ccnet/seafile/seahub databases +''' + return Utils.ask_question(question, + key='1 or 2', + note=note, + validate=validate) + + def validate_mysql_host(self, host): + if not re.match(r'^[a-zA-Z0-9_\-\.]+$', host): + raise InvalidAnswer('%s is not a valid host' % Utils.highlight(host)) + + if host == 'localhost': + host = '127.0.0.1' + return host + + def ask_mysql_host(self): + question = 'What is the host of mysql server?' + key = 'mysql server host' + default = 'localhost' + self.mysql_host = Utils.ask_question(question, + key=key, + default=default, + validate=self.validate_mysql_host) + + def validate_mysql_user_host(self, host): + MYSQL_HOST_RE = re.compile(r'^(%|[^.].+\..+[^.])$') + if not MYSQL_HOST_RE.match(host): + raise InvalidAnswer('invalid mysql user host: {}'.format(host)) + return host + + def ask_mysql_user_host(self): + self.seafile_mysql_userhost = Utils.ask_question( + 'From which hosts could the mysql account be used?', + key='mysql user host', + default='%', + validate=self.validate_mysql_user_host + ) + + def ask_mysql_port(self): + question = 'What is the port of mysql server?' + key = 'mysql server port' + default = '3306' + port = Utils.ask_question(question, + key=key, + default=default, + validate=Utils.validate_port) + + # self.check_mysql_server(host, port) + self.mysql_port = port + + def ask_mysql_host_port(self): + self.ask_mysql_host() + if self.mysql_host != '127.0.0.1': + self.ask_mysql_user_host() + self.ask_mysql_port() + + def check_mysql_server(self, host, port): + print '\nverifying mysql server running ... ', + try: + dummy = MySQLdb.connect(host=host, port=port) + except Exception: + print + raise InvalidAnswer('Failed to connect to mysql server at "%s:%s"' \ + % (host, port)) + + print 'done' + + def check_mysql_user(self, user, password, host=None): + print '\nverifying password of user %s ... ' % user, + kwargs = dict(host=host or self.mysql_host, + port=self.mysql_port, + user=user, + passwd=password) + + try: + conn = MySQLdb.connect(**kwargs) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + raise InvalidAnswer('Failed to connect to mysql server using user "%s" and password "***": %s' \ + % (user, e.args[1])) + else: + raise InvalidAnswer('Failed to connect to mysql server using user "%s" and password "***": %s' \ + % (user, e)) + + print 'done' + return conn + + def create_seahub_admin(self): + try: + conn = MySQLdb.connect(host=self.mysql_host, + port=self.mysql_port, + user=self.seafile_mysql_user, + passwd=self.seafile_mysql_password, + db=self.ccnet_db_name) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to connect to mysql database %s: %s' % (self.ccnet_db_name, e.args[1])) + else: + Utils.error('Failed to connect to mysql database %s: %s' % (self.ccnet_db_name, e)) + + cursor = conn.cursor() + sql = '''\ +CREATE TABLE IF NOT EXISTS EmailUser (id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT, email VARCHAR(255), passwd CHAR(64), is_staff BOOL NOT NULL, is_active BOOL NOT NULL, ctime BIGINT, UNIQUE INDEX (email)) ENGINE=INNODB''' + + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to create ccnet user table: %s' % e.args[1]) + else: + Utils.error('Failed to create ccnet user table: %s' % e) + + sql = '''REPLACE INTO EmailUser(email, passwd, is_staff, is_active, ctime) VALUES ('%s', '%s', 1, 1, 0)''' \ + % (seahub_config.admin_email, seahub_config.hashed_admin_password()) + + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to create admin user: %s' % e.args[1]) + else: + Utils.error('Failed to create admin user: %s' % e) + + conn.commit() + + def ask_questions(self): + '''Ask questions and do database operations''' + raise NotImplementedError + + +class NewDBConfigurator(AbstractDBConfigurator): + '''Handles the case of creating new mysql databases for ccnet/seafile/seahub''' + def __init__(self): + AbstractDBConfigurator.__init__(self) + + def ask_questions(self): + self.ask_mysql_host_port() + + self.ask_root_password() + self.ask_seafile_mysql_user_password() + + self.ask_db_names() + + def generate(self): + #if not self.mysql_user_exists(self.seafile_mysql_user): + # self.create_user() + #self.create_databases() + pass + + def validate_root_passwd(self, password): + try: + self.root_conn = self.check_mysql_user('root', password) + except InvalidAnswer: + # For MariaDB on Ubuntu 16.04, the msyql root user can only be + # accessed from localhost with unix socket. So we retry with + # localhost when failing with 127.0.0.1. + if self.mysql_host == '127.0.0.1': + self.root_conn = self.check_mysql_user('root', password, host='localhost') + else: + raise + return password + + def ask_root_password(self): + question = 'What is the password of the mysql root user?' + key = 'root password' + self.root_password = Utils.ask_question(question, + key=key, + validate=self.validate_root_passwd, + password=True) + + def mysql_user_exists(self, user): + cursor = self.root_conn.cursor() + + sql = '''SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '%s' and host = '%s')''' % \ + (user, self.seafile_mysql_userhost) + + try: + cursor.execute(sql) + return cursor.fetchall()[0][0] + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to check mysql user %s@%s: %s' % \ + (user, self.seafile_mysql_userhost, e.args[1])) + else: + Utils.error('Failed to check mysql user %s@%s: %s' % \ + (user, self.seafile_mysql_userhost, e)) + finally: + cursor.close() + + + def ask_seafile_mysql_user_password(self): + def validate(user): + if user == 'root': + raise InvalidAnswer( + 'Using mysql "root" user is not allowed for security reasons. Please specify a different database user.' + ) + else: + question = 'Enter the password for mysql user "%s":' % Utils.highlight(user) + key = 'password for %s' % user + password = Utils.ask_question(question, key=key, password=True) + # If the user already exists, check the password here + #if self.mysql_user_exists(user): + # self.check_mysql_user(user, password) + self.seafile_mysql_password = password + + return user + + + question = 'Enter the name for mysql user of seafile. It would be created if not exists.' + key = 'mysql user for seafile' + default = 'seafile' + self.seafile_mysql_user = Utils.ask_question(question, + key=key, + default=default, + validate=validate) + + def ask_db_name(self, program, default): + question = 'Enter the database name for %s:' % program + key = '%s database' % program + return Utils.ask_question(question, + key=key, + default=default, + validate=self.validate_db_name) + + def ask_db_names(self): + self.ccnet_db_name = self.ask_db_name('ccnet-server', 'ccnet-db') + self.seafile_db_name = self.ask_db_name('seafile-server', 'seafile-db') + self.seahub_db_name = self.ask_db_name('seahub', 'seahub-db') + + def validate_db_name(self, db_name): + return db_name + + def create_user(self): + cursor = self.root_conn.cursor() + sql = '''CREATE USER '{}'@'{}' IDENTIFIED BY '{}' '''.format( + self.seafile_mysql_user, + self.seafile_mysql_userhost, + self.seafile_mysql_password + ) + + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to create mysql user {}@{}: {}'.format(self.seafile_mysql_user, self.seafile_mysql_userhost, e.args[1])) + else: + Utils.error('Failed to create mysql user {}@{}: {}'.format(self.seafile_mysql_user, self.seafile_mysql_userhost, e)) + finally: + cursor.close() + + + def create_db(self, db_name): + cursor = self.root_conn.cursor() + sql = '''CREATE DATABASE IF NOT EXISTS `%s` CHARACTER SET UTF8''' \ + % db_name + + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to create database %s: %s' % (db_name, e.args[1])) + else: + Utils.error('Failed to create database %s: %s' % (db_name, e)) + finally: + cursor.close() + + def grant_db_permission(self, db_name): + cursor = self.root_conn.cursor() + sql = '''GRANT ALL PRIVILEGES ON `{}`.* to `{}`@`{}` '''.format( + db_name, + self.seafile_mysql_user, + self.seafile_mysql_userhost + ) + + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to grant permission of database %s: %s' % (db_name, e.args[1])) + else: + Utils.error('Failed to grant permission of database %s: %s' % (db_name, e)) + finally: + cursor.close() + + def create_databases(self): + self.create_db(self.ccnet_db_name) + self.create_db(self.seafile_db_name) + self.create_db(self.seahub_db_name) + + if self.seafile_mysql_user != 'root': + self.grant_db_permission(self.ccnet_db_name) + self.grant_db_permission(self.seafile_db_name) + self.grant_db_permission(self.seahub_db_name) + + +class ExistingDBConfigurator(AbstractDBConfigurator): + '''Handles the case of use existing mysql databases for ccnet/seafile/seahub''' + def __init__(self): + AbstractDBConfigurator.__init__(self) + self.use_existing_db = True + + def ask_questions(self): + self.ask_mysql_host_port() + + self.ask_existing_mysql_user_password() + + self.ccnet_db_name = self.ask_db_name('ccnet') + self.seafile_db_name = self.ask_db_name('seafile') + self.seahub_db_name = self.ask_db_name('seahub') + + def generate(self): + pass + + def ask_existing_mysql_user_password(self): + def validate(user): + if user == 'root': + raise InvalidAnswer( + 'Using root is not allowed for security reasons. Please specify a different database user.' + ) + question = 'What is the password for mysql user "%s"?' % Utils.highlight(user) + key = 'password for %s' % user + password = Utils.ask_question(question, key=key, password=True) + #self.check_mysql_user(user, password) + self.seafile_mysql_password = password + return user + + question = 'Which mysql user to use for seafile?' + key = 'mysql user for seafile' + self.seafile_mysql_user = Utils.ask_question(question, + key=key, + validate=validate) + + def validate_db_name(self, db_name): + self.check_user_db_access(db_name) + return db_name + + def ask_db_name(self, program): + question = 'Enter the existing database name for %s:' % program + key = '%s database' % program + return Utils.ask_question(question, + key=key, + validate=self.validate_db_name) + + def check_user_db_access(self, db_name): + user = self.seafile_mysql_user + password = self.seafile_mysql_password + + print '\nverifying user "%s" access to database %s ... ' % (user, db_name), + try: + conn = MySQLdb.connect(host=self.mysql_host, + port=self.mysql_port, + user=user, + passwd=password, + db=db_name) + + cursor = conn.cursor() + cursor.execute('show tables') + cursor.close() + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + raise InvalidAnswer('Failed to access database %s using user "%s" and password "***": %s' \ + % (db_name, user, e.args[1])) + else: + raise InvalidAnswer('Failed to access database %s using user "%s" and password "***": %s' \ + % (db_name, user, e)) + + print 'done' + + return conn + + +class CcnetConfigurator(AbstractConfigurator): + SERVER_NAME_REGEX = r'^[a-zA-Z0-9_\-]{3,15}$' + SERVER_IP_OR_DOMAIN_REGEX = r'^[^.].+\..+[^.]$' + + def __init__(self): + '''Initialize default values of ccnet configuration''' + AbstractConfigurator.__init__(self) + self.ccnet_dir = os.path.join(env_mgr.top_dir, 'ccnet') + self.port = 10001 + self.server_name = None + self.ip_or_domain = None + self.ccnet_conf = os.path.join(env_mgr.central_config_dir, 'ccnet.conf') + + def ask_questions(self): + if not self.server_name: + self.ask_server_name() + if not self.ip_or_domain: + self.ask_server_ip_or_domain() + # self.ask_port() + + def generate(self): + print 'Generating ccnet configuration ...\n' + ccnet_init = os.path.join(env_mgr.bin_dir, 'ccnet-init') + argv = [ + ccnet_init, + '-F', env_mgr.central_config_dir, + '--config-dir', self.ccnet_dir, + '--name', self.server_name, + '--host', self.ip_or_domain, + ] + + if Utils.run_argv(argv, env=env_mgr.get_binary_env()) != 0: + Utils.error('Failed to generate ccnet configuration') + + time.sleep(1) + self.generate_db_conf() + + def generate_db_conf(self): + config = Utils.read_config(self.ccnet_conf) + # [Database] + # ENGINE= + # HOST= + # USER= + # PASSWD= + # DB= + db_section = 'Database' + if not config.has_section(db_section): + config.add_section(db_section) + config.set(db_section, 'ENGINE', 'mysql') + config.set(db_section, 'HOST', db_config.mysql_host) + config.set(db_section, 'PORT', db_config.mysql_port) + config.set(db_section, 'USER', db_config.seafile_mysql_user) + config.set(db_section, 'PASSWD', db_config.seafile_mysql_password) + config.set(db_section, 'DB', db_config.ccnet_db_name) + config.set(db_section, 'CONNECTION_CHARSET', 'utf8') + + Utils.write_config(config, self.ccnet_conf) + + def validate_server_name(self, name): + if not re.match(self.SERVER_NAME_REGEX, name): + raise InvalidAnswer('%s is not a valid name' % Utils.highlight(name)) + return name + + def ask_server_name(self): + question = 'What is the name of the server? It will be displayed on the client.' + key = 'server name' + note = '3 - 15 letters or digits' + self.server_name = Utils.ask_question(question, + key=key, + note=note, + validate=self.validate_server_name) + + def validate_server_ip(self, ip_or_domain): + if not re.match(self.SERVER_IP_OR_DOMAIN_REGEX, ip_or_domain): + raise InvalidAnswer('%s is not a valid ip or domain' % ip_or_domain) + return ip_or_domain + + def ask_server_ip_or_domain(self): + question = 'What is the ip or domain of the server?' + key = 'This server\'s ip or domain' + note = 'For example: www.mycompany.com, 192.168.1.101' + self.ip_or_domain = Utils.ask_question(question, + key=key, + note=note, + validate=self.validate_server_ip) + + def ask_port(self): + def validate(port): + return Utils.validate_port(port) + + question = 'Which port do you want to use for the ccnet server?' + key = 'ccnet server port' + default = 10001 + self.port = Utils.ask_question(question, + key=key, + default=default, + validate=validate) + + +class SeafileConfigurator(AbstractConfigurator): + def __init__(self): + AbstractConfigurator.__init__(self) + self.seafile_dir = None + self.port = 12001 + self.fileserver_port = None + self.seafile_conf = os.path.join(env_mgr.central_config_dir, 'seafile.conf') + + def ask_questions(self): + if not self.seafile_dir: + self.ask_seafile_dir() + # self.ask_port() + if not self.fileserver_port: + self.ask_fileserver_port() + + def generate(self): + print 'Generating seafile configuration ...\n' + seafserv_init = os.path.join(env_mgr.bin_dir, 'seaf-server-init') + argv = [ + seafserv_init, + '-F', env_mgr.central_config_dir, + '--seafile-dir', self.seafile_dir, + '--fileserver-port', str(self.fileserver_port), + ] + + if Utils.run_argv(argv, env=env_mgr.get_binary_env()) != 0: + Utils.error('Failed to generate ccnet configuration') + + time.sleep(1) + self.generate_db_conf() + self.write_seafile_ini() + print 'done' + + def generate_db_conf(self): + config = Utils.read_config(self.seafile_conf) + # [database] + # type= + # host= + # user= + # password= + # db_name= + # unix_socket= + db_section = 'database' + if not config.has_section(db_section): + config.add_section(db_section) + config.set(db_section, 'type', 'mysql') + config.set(db_section, 'host', db_config.mysql_host) + config.set(db_section, 'port', db_config.mysql_port) + config.set(db_section, 'user', db_config.seafile_mysql_user) + config.set(db_section, 'password', db_config.seafile_mysql_password) + config.set(db_section, 'db_name', db_config.seafile_db_name) + config.set(db_section, 'connection_charset', 'utf8') + + Utils.write_config(config, self.seafile_conf) + + def validate_seafile_dir(self, path): + if os.path.exists(path): + raise InvalidAnswer('%s already exists' % Utils.highlight(path)) + return path + + def ask_seafile_dir(self): + question = 'Where do you want to put your seafile data?' + key = 'seafile-data' + note = 'Please use a volume with enough free space' + default = os.path.join(env_mgr.top_dir, 'seafile-data') + self.seafile_dir = Utils.ask_question(question, + key=key, + note=note, + default=default, + validate=self.validate_seafile_dir) + + def ask_port(self): + def validate(port): + port = Utils.validate_port(port) + if port == ccnet_config.port: + raise InvalidAnswer('%s is used by ccnet server, choose another one' \ + % Utils.highlight(port)) + return port + + question = 'Which port do you want to use for the seafile server?' + key = 'seafile server port' + default = 12001 + self.port = Utils.ask_question(question, + key=key, + default=default, + validate=validate) + + def ask_fileserver_port(self): + question = 'Which port do you want to use for the seafile fileserver?' + key = 'seafile fileserver port' + default = 8082 + self.fileserver_port = Utils.ask_question(question, + key=key, + default=default, + validate=Utils.validate_port) + + def write_seafile_ini(self): + seafile_ini = os.path.join(ccnet_config.ccnet_dir, 'seafile.ini') + with open(seafile_ini, 'w') as fp: + fp.write(self.seafile_dir) + +class SeahubConfigurator(AbstractConfigurator): + def __init__(self): + AbstractConfigurator.__init__(self) + self.admin_email = '' + self.admin_password = '' + self.seahub_settings_py = os.path.join(env_mgr.central_config_dir, 'seahub_settings.py') + + def hashed_admin_password(self): + return hashlib.sha1(self.admin_password).hexdigest() # pylint: disable=E1101 + + def ask_questions(self): + pass + + def generate(self): + '''Generating seahub_settings.py''' + print 'Generating seahub configuration ...\n' + time.sleep(1) + with open(self.seahub_settings_py, 'w') as fp: + self.write_utf8_comment(fp) + fp.write('\n') + self.write_secret_key(fp) + fp.write('\n') + self.write_database_config(fp) + + def write_utf8_comment(self, fp): + fp.write('# -*- coding: utf-8 -*-') + + def write_secret_key(self, fp): + script = os.path.join(env_mgr.install_path, 'seahub/tools/secret_key_generator.py') + cmd = [ + Utils.get_python_executable(), + script, + ] + key = Utils.get_command_output(cmd).strip() + fp.write('SECRET_KEY = "%s"' % key) + + def write_database_config(self, fp): + template = '''\ +\nDATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '%(name)s', + 'USER': '%(username)s', + 'PASSWORD': '%(password)s', + 'HOST': '%(host)s', + 'PORT': '%(port)s' + } +} + +''' + text = template % dict(name=db_config.seahub_db_name, + username=db_config.seafile_mysql_user, + password=db_config.seafile_mysql_password, + host=db_config.mysql_host, + port=db_config.mysql_port) + + fp.write(text) + + def ask_admin_email(self): + print + print '----------------------------------------' + print 'Now let\'s create the admin account' + print '----------------------------------------' + def validate(email): + # whitespace is not allowed + if re.match(r'[\s]', email): + raise InvalidAnswer('%s is not a valid email address' % Utils.highlight(email)) + # must be a valid email address + if not re.match(r'^.+@.*\..+$', email): + raise InvalidAnswer('%s is not a valid email address' % Utils.highlight(email)) + + return email + + key = 'admin email' + question = 'What is the ' + Utils.highlight('email') + ' for the admin account?' + self.admin_email = Utils.ask_question(question, + key=key, + validate=validate) + + def ask_admin_password(self): + def validate(password): + key = 'admin password again' + question = 'Enter the ' + Utils.highlight('password again:') + password_again = Utils.ask_question(question, + key=key, + password=True) + + if password_again != password: + raise InvalidAnswer('password mismatch') + + return password + + key = 'admin password' + question = 'What is the ' + Utils.highlight('password') + ' for the admin account?' + self.admin_password = Utils.ask_question(question, + key=key, + password=True, + validate=validate) + + def do_syncdb(self): + print '----------------------------------------' + print 'Now creating seahub database tables ...\n' + print '----------------------------------------' + + try: + conn = MySQLdb.connect(host=db_config.mysql_host, + port=db_config.mysql_port, + user=db_config.seafile_mysql_user, + passwd=db_config.seafile_mysql_password, + db=db_config.seahub_db_name) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to connect to mysql database %s: %s' % (db_config.seahub_db_name, e.args[1])) + else: + Utils.error('Failed to connect to mysql database %s: %s' % (db_config.seahub_db_name, e)) + + cursor = conn.cursor() + + sql_file = os.path.join(env_mgr.install_path, 'seahub', 'sql', 'mysql.sql') + with open(sql_file, 'r') as fp: + content = fp.read() + + sqls = [line.strip() for line in content.split(';') if line.strip()] + for sql in sqls: + try: + cursor.execute(sql) + except Exception, e: + if isinstance(e, MySQLdb.OperationalError): + Utils.error('Failed to init seahub database: %s' % e.args[1]) + else: + Utils.error('Failed to init seahub database: %s' % e) + + conn.commit() + + def prepare_avatar_dir(self): + # media_dir=${INSTALLPATH}/seahub/media + # orig_avatar_dir=${INSTALLPATH}/seahub/media/avatars + # dest_avatar_dir=${TOPDIR}/seahub-data/avatars + + # if [[ ! -d ${dest_avatar_dir} ]]; then + # mkdir -p "${TOPDIR}/seahub-data" + # mv "${orig_avatar_dir}" "${dest_avatar_dir}" + # ln -s ../../../seahub-data/avatars ${media_dir} + # fi + + try: + media_dir = os.path.join(env_mgr.install_path, 'seahub', 'media') + orig_avatar_dir = os.path.join(media_dir, 'avatars') + + seahub_data_dir = os.path.join(env_mgr.top_dir, 'seahub-data') + dest_avatar_dir = os.path.join(seahub_data_dir, 'avatars') + + if os.path.exists(dest_avatar_dir): + return + + if not os.path.exists(seahub_data_dir): + os.mkdir(seahub_data_dir) + + shutil.move(orig_avatar_dir, dest_avatar_dir) + os.symlink('../../../seahub-data/avatars', orig_avatar_dir) + except Exception, e: + Utils.error('Failed to prepare seahub avatars dir: %s' % e) + +class SeafDavConfigurator(AbstractConfigurator): + def __init__(self): + AbstractConfigurator.__init__(self) + self.seafdav_conf = None + + def ask_questions(self): + pass + + def generate(self): + self.seafdav_conf = os.path.join(env_mgr.central_config_dir, 'seafdav.conf') + text = ''' +[WEBDAV] +enabled = false +port = 8080 +fastcgi = false +share_name = / +''' + + with open(self.seafdav_conf, 'w') as fp: + fp.write(text) + +class ProfessionalConfigurator(AbstractConfigurator): + '''Seafile Pro related configuration''' + def __init__(self): + AbstractConfigurator.__init__(self) + self.pro_py = os.path.join(env_mgr.install_path, 'pro', 'pro.py') + self.pro_data_dir = os.path.join(env_mgr.top_dir, 'pro-data') + + def ask_questions(self): + pass + + def generate(self): + argv = [ + Utils.get_python_executable(), + self.pro_py, + 'setup', + '--mysql', + '--mysql_host=%s' % db_config.mysql_host, + '--mysql_port=%s' % db_config.mysql_port, + '--mysql_user=%s' % db_config.seafile_mysql_user, + '--mysql_password=%s' % db_config.seafile_mysql_password, + '--mysql_db=%s' % db_config.seahub_db_name, + ] + if Utils.run_argv(argv, env=env_mgr.get_seahub_env()) != 0: + Utils.error('Failed to generate seafile pro configuration') + +class UserManualHandler(object): + def __init__(self): + self.src_docs_dir = os.path.join(env_mgr.install_path, 'seafile', 'docs') + self.library_template_dir = None + + def copy_user_manuals(self): + self.library_template_dir = os.path.join(seafile_config.seafile_dir, 'library-template') + Utils.must_mkdir(self.library_template_dir) + + pattern = os.path.join(self.src_docs_dir, '*.doc') + + for doc in glob.glob(pattern): + Utils.must_copy(doc, self.library_template_dir) + +def report_config(): + print + print '---------------------------------' + print 'This is your configuration' + print '---------------------------------' + print + + template = '''\ + server name: %(server_name)s + server ip/domain: %(ip_or_domain)s + + seafile data dir: %(seafile_dir)s + fileserver port: %(fileserver_port)s + + database: %(use_existing_db)s + ccnet database: %(ccnet_db_name)s + seafile database: %(seafile_db_name)s + seahub database: %(seahub_db_name)s + database user: %(db_user)s + +''' + config = { + 'server_name' : ccnet_config.server_name, + 'ip_or_domain' : ccnet_config.ip_or_domain, + + 'seafile_dir' : seafile_config.seafile_dir, + 'fileserver_port' : seafile_config.fileserver_port, + + 'admin_email' : seahub_config.admin_email, + + + 'use_existing_db': 'use existing' if db_config.use_existing_db else 'create new', + 'ccnet_db_name': db_config.ccnet_db_name, + 'seafile_db_name': db_config.seafile_db_name, + 'seahub_db_name': db_config.seahub_db_name, + 'db_user': db_config.seafile_mysql_user + } + + print template % config + + if need_pause: + print + print '---------------------------------' + print 'Press ENTER to continue, or Ctrl-C to abort' + print '---------------------------------' + + raw_input() + + +def set_file_perm(): + filemode = 0600 + dirmode = 0700 + files = [ + seahub_config.seahub_settings_py, + ] + dirs = [ + env_mgr.central_config_dir, + ccnet_config.ccnet_dir, + seafile_config.seafile_dir, + seahub_config.seahub_settings_py, + ] + for fpath in files: + os.chmod(fpath, filemode) + for dpath in dirs: + os.chmod(dpath, dirmode) + +env_mgr = EnvManager() +ccnet_config = CcnetConfigurator() +seafile_config = SeafileConfigurator() +seafdav_config = SeafDavConfigurator() +seahub_config = SeahubConfigurator() +user_manuals_handler = UserManualHandler() +pro_config = ProfessionalConfigurator() +# Would be created after AbstractDBConfigurator.ask_use_existing_db() +db_config = None +need_pause = True + +def get_param_val(arg, env, default=None): + return arg or os.environ.get(env, default) + +def check_params(args): + server_name = 'seafile' + ccnet_config.server_name = ccnet_config.validate_server_name(server_name) + + server_ip = get_param_val(args.server_ip, 'SERVER_IP', '127.0.0.1') + ccnet_config.ip_or_domain = ccnet_config.validate_server_ip(server_ip) + + fileserver_port = get_param_val(args.fileserver_port, 'FILESERVER_PORT', '8082') + seafile_config.fileserver_port = Utils.validate_port(fileserver_port) + + seafile_dir = get_param_val(args.seafile_dir, 'SEAFILE_DIR', + os.path.join(env_mgr.top_dir, 'seafile-data')) + seafile_config.seafile_dir = seafile_config.validate_seafile_dir(seafile_dir) + + global db_config + + use_existing_db = get_param_val(args.use_existing_db, 'USE_EXISTING_DB', '0') + # pylint: disable=redefined-variable-type + if use_existing_db == '0': + db_config = NewDBConfigurator() + elif use_existing_db == '1': + db_config = ExistingDBConfigurator() + else: + raise InvalidParams('Invalid use existing db parameter, the value can only be 0 or 1') + + mysql_host = get_param_val(args.mysql_host, 'MYSQL_HOST', '127.0.0.1') + if not mysql_host: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing mysql host parameter') + db_config.mysql_host = db_config.validate_mysql_host(mysql_host) + + mysql_port = get_param_val(args.mysql_port, 'MYSQL_PORT', '3306') + db_config.mysql_port = Utils.validate_port(mysql_port) + + mysql_user = get_param_val(args.mysql_user, 'MYSQL_USER') + if not mysql_user: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing mysql user name parameter') + + mysql_user_passwd = get_param_val(args.mysql_user_passwd, 'MYSQL_USER_PASSWD') + if not mysql_user_passwd: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing mysql user password parameter') + + ccnet_db = get_param_val(args.ccnet_db, 'CCNET_DB', 'ccnet_db') + if not ccnet_db: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing ccnet db name parameter') + + seafile_db = get_param_val(args.seafile_db, 'SEAFILE_DB', 'seafile_db') + if not seafile_db: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing seafile db name parameter') + + seahub_db = get_param_val(args.seahub_db, 'SEAHUB_DB', 'seahub_db') + if not seahub_db: + raise InvalidParams('Incomplete mysql configuration parameters, ' \ + 'missing seahub db name parameter') + + mysql_user_host = get_param_val(args.mysql_user_host, 'MYSQL_USER_HOST') + mysql_root_passwd = get_param_val(args.mysql_root_passwd, 'MYSQL_ROOT_PASSWD') + + if db_config.use_existing_db: + db_config.check_mysql_user(mysql_user, mysql_user_passwd) + db_config.seafile_mysql_user = mysql_user + db_config.seafile_mysql_password = mysql_user_passwd + db_config.ccnet_db_name = db_config.validate_db_name(ccnet_db) + db_config.seafile_db_name = db_config.validate_db_name(seafile_db) + db_config.seahub_db_name = db_config.validate_db_name(seahub_db) + else: + if db_config.mysql_host != '127.0.0.1' and not mysql_user_host: + raise InvalidParams('mysql user host parameter is missing in creating new db mode') + if not mysql_user_host: + db_config.seafile_mysql_userhost = 'localhost' + else: + db_config.seafile_mysql_userhost = db_config.validate_mysql_user_host(mysql_user_host) + + if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ: + raise InvalidParams('mysql root password parameter is missing in creating new db mode') + db_config.root_password = mysql_root_passwd + + if mysql_user == 'root': + db_config.seafile_mysql_user = 'root' + db_config.seafile_mysql_password = db_config.root_password + else: + #if db_config.mysql_user_exists(mysql_user): + # db_config.check_mysql_user(mysql_user, mysql_user_passwd) + db_config.seafile_mysql_user = mysql_user + db_config.seafile_mysql_password = mysql_user_passwd + db_config.ccnet_db_name = ccnet_db + db_config.seafile_db_name = seafile_db + db_config.seahub_db_name = seahub_db + + global need_pause + need_pause = False + + +def main(): + if len(sys.argv) > 2 and sys.argv[1] == 'auto': + sys.argv.remove('auto') + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--server-name', help='server name') + parser.add_argument('-i', '--server-ip', help='server ip or domain') + parser.add_argument('-p', '--fileserver-port', help='fileserver port') + parser.add_argument('-d', '--seafile-dir', help='seafile dir to store seafile data') + parser.add_argument('-e', '--use-existing-db', + help='use mysql existing dbs or create new dbs, ' + '0: create new dbs 1: use existing dbs') + parser.add_argument('-o', '--mysql-host', help='mysql host') + parser.add_argument('-t', '--mysql-port', help='mysql port') + parser.add_argument('-u', '--mysql-user', help='mysql user name') + parser.add_argument('-w', '--mysql-user-passwd', help='mysql user password') + parser.add_argument('-q', '--mysql-user-host', help='mysql user host') + parser.add_argument('-r', '--mysql-root-passwd', help='mysql root password') + parser.add_argument('-c', '--ccnet-db', help='ccnet db name') + parser.add_argument('-s', '--seafile-db', help='seafile db name') + parser.add_argument('-b', '--seahub-db', help='seahub db name') + + args = parser.parse_args() + + try: + check_params(args) + except (InvalidAnswer, InvalidParams) as e: + print Utils.highlight('\n%s\n' % e) + sys.exit(-1) + + global db_config + + if need_pause: + Utils.welcome() + warnings.filterwarnings('ignore', category=MySQLdb.Warning) + + env_mgr.check_pre_condiction() + + # Part 1: collect configuration + ccnet_config.ask_questions() + seafile_config.ask_questions() + seahub_config.ask_questions() + pro_config.ask_questions() + + # pylint: disable=redefined-variable-type + if not db_config: + if AbstractDBConfigurator.ask_use_existing_db(): + db_config = ExistingDBConfigurator() + else: + db_config = NewDBConfigurator() + + db_config.ask_questions() + + report_config() + + # Part 2: generate configuration + db_config.generate() + ccnet_config.generate() + seafile_config.generate() + seafdav_config.generate() + seahub_config.generate() + pro_config.generate() + + #seahub_config.do_syncdb() + seahub_config.prepare_avatar_dir() + # db_config.create_seahub_admin() + user_manuals_handler.copy_user_manuals() + #create_seafile_server_symlink() + + #set_file_perm() + set_file_perm() + + report_success() + +def report_success(): + message = '''\ + + +----------------------------------------------------------------- +Your seafile server configuration has been finished successfully. +----------------------------------------------------------------- + +run seafile server: ./seafile.sh { start | stop | restart } +run seahub server: ./seahub.sh { start | stop | restart } + +----------------------------------------------------------------- +If you are behind a firewall, remember to allow input/output of these tcp ports: +----------------------------------------------------------------- + +port of seafile fileserver: %(fileserver_port)s +port of seahub: 8000 + +When problems occur, Refer to + + %(server_manual_http)s + +for information. + +''' + + print message % dict(fileserver_port=seafile_config.fileserver_port, + server_manual_http=SERVER_MANUAL_HTTP) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print + print Utils.highlight('The setup process is aborted') + print diff --git a/cluster/image/pro_seafile/scripts/ssl.sh b/cluster/image/pro_seafile/scripts/ssl.sh new file mode 100755 index 0000000..931219a --- /dev/null +++ b/cluster/image/pro_seafile/scripts/ssl.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e + +ssldir=${1:?"error params"} +domain=${2:?"error params"} + +letsencryptdir=$ssldir/letsencrypt +letsencrypt_script=$letsencryptdir/acme_tiny.py + +ssl_account_key=${domain}.account.key +ssl_csr=${domain}.csr +ssl_key=${domain}.key +ssl_crt=${domain}.crt + +mkdir -p /var/www/challenges && chmod -R 777 /var/www/challenges +mkdir -p ssldir + +if ! [[ -d $letsencryptdir ]]; then + git clone git://github.com/diafygi/acme-tiny.git $letsencryptdir +else + cd $letsencryptdir + git pull origin master:master +fi + +cd $ssldir + +if [[ ! -e ${ssl_account_key} ]]; then + openssl genrsa 4096 > ${ssl_account_key} +fi + +if [[ ! -e ${ssl_key} ]]; then + openssl genrsa 4096 > ${ssl_key} +fi + +if [[ ! -e ${ssl_csr} ]]; then + openssl req -new -sha256 -key ${ssl_key} -subj "/CN=$domain" > $ssl_csr +fi + +python $letsencrypt_script --account-key ${ssl_account_key} --csr $ssl_csr --acme-dir /var/www/challenges/ > ./signed.crt +curl -sSL -o intermediate.pem https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.pem +cat signed.crt intermediate.pem > ${ssl_crt} + +nginx -s reload + +echo "Nginx reloaded." diff --git a/cluster/image/pro_seafile/scripts/start.py b/cluster/image/pro_seafile/scripts/start.py new file mode 100755 index 0000000..99b2951 --- /dev/null +++ b/cluster/image/pro_seafile/scripts/start.py @@ -0,0 +1,61 @@ +import os +import time +import json +import argparse +from os.path import join, exists, dirname + +from upgrade import check_upgrade +from utils import call, get_conf, get_script, get_command_output, get_install_dir + +installdir = get_install_dir() +topdir = dirname(installdir) + +def watch_controller(): + maxretry = 4 + retry = 0 + while retry < maxretry: + controller_pid = get_command_output('ps aux | grep seafile-controller | grep -v grep || true').strip() + garbage_collector_pid = get_command_output('ps aux | grep /scripts/gc.sh | grep -v grep || true').strip() + if not controller_pid and not garbage_collector_pid: + retry += 1 + else: + retry = 0 + time.sleep(5) + print 'seafile controller exited unexpectedly.' + sys.exit(1) + +def main(args): + call('/scripts/create_data_links.sh') + check_upgrade() + os.chdir(installdir) + call('service nginx start &') + + admin_pw = { + 'email': get_conf('SEAFILE_ADMIN_EMAIL', 'me@example.com'), + 'password': get_conf('SEAFILE_ADMIN_PASSWORD', 'asecret'), + } + password_file = join(topdir, 'conf', 'admin.txt') + with open(password_file, 'w+') as fp: + json.dump(admin_pw, fp) + + + try: + call('{} start'.format(get_script('seafile.sh'))) + call('{} start'.format(get_script('seahub.sh'))) + if args.mode == 'backend': + call('{} start'.format(get_script('seafile-background-tasks.sh'))) + finally: + if exists(password_file): + os.unlink(password_file) + + print 'seafile server is running now.' + try: + watch_controller() + except KeyboardInterrupt: + print 'Stopping seafile server.' + sys.exit(0) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Seafile cluster start script') + parser.add_argument('--mode') + main(parser.parse_args()) diff --git a/cluster/image/pro_seafile/scripts/start.sh b/cluster/image/pro_seafile/scripts/start.sh new file mode 100755 index 0000000..4f9b2fb --- /dev/null +++ b/cluster/image/pro_seafile/scripts/start.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +function start-front-end() { + python /scripts/start.py +} + +function start-back-end() { + python /scripts/start.py --mode backend +} + +case $1 in + "front-end" ) + start-front-end + ;; + "back-end" ) + start-back-end + ;; +esac diff --git a/cluster/image/pro_seafile/scripts/upgrade.py b/cluster/image/pro_seafile/scripts/upgrade.py new file mode 100755 index 0000000..9d2e4d3 --- /dev/null +++ b/cluster/image/pro_seafile/scripts/upgrade.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +This script is used to run proper upgrade scripts automatically. +""" + +import json +import re +import glob +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys +import time + +from utils import ( + call, get_install_dir, get_script, get_command_output, replace_file_pattern, + read_version_stamp, wait_for_mysql, update_version_stamp, loginfo +) + +installdir = get_install_dir() +topdir = dirname(installdir) + +def collect_upgrade_scripts(from_version, to_version): + """ + Give the current installed version, calculate which upgrade scripts we need + to run to upgrade it to the latest verison. + + For example, given current version 5.0.1 and target version 6.1.0, and these + upgrade scripts: + + upgrade_4.4_5.0.sh + upgrade_5.0_5.1.sh + upgrade_5.1_6.0.sh + upgrade_6.0_6.1.sh + + We need to run upgrade_5.0_5.1.sh, upgrade_5.1_6.0.sh, and upgrade_6.0_6.1.sh. + """ + from_major_ver = '.'.join(from_version.split('.')[:2]) + to_major_ver = '.'.join(to_version.split('.')[:2]) + + scripts = [] + for fn in sorted(glob.glob(join(installdir, 'upgrade', 'upgrade_*_*.sh'))): + va, vb = parse_upgrade_script_version(fn) + if va >= from_major_ver and vb <= to_major_ver: + scripts.append(fn) + return scripts + +def parse_upgrade_script_version(script): + script = basename(script) + m = re.match(r'upgrade_([0-9+.]+)_([0-9+.]+).sh', basename(script)) + return m.groups() + +def check_upgrade(): + last_version = read_version_stamp() + current_version = os.environ['SEAFILE_VERSION'] + if last_version == current_version: + return + + scripts_to_run = collect_upgrade_scripts(from_version=last_version, to_version=current_version) + for script in scripts_to_run: + loginfo('Running scripts {}'.format(script)) + # Here we use a trick: use a version stamp like 6.1.0 to prevent running + # all upgrade scripts before 6.1 again (because 6.1 < 6.1.0 in python) + new_version = parse_upgrade_script_version(script)[1] + '.0' + + replace_file_pattern(script, 'read dummy', '') + call(script) + + update_version_stamp(new_version) + + update_version_stamp(current_version) + +def main(): + wait_for_mysql() + + os.chdir(installdir) + check_upgrade() + +if __name__ == '__main__': + main() diff --git a/cluster/image/pro_seafile/scripts/utils/__init__.py b/cluster/image/pro_seafile/scripts/utils/__init__.py new file mode 100644 index 0000000..ee691f5 --- /dev/null +++ b/cluster/image/pro_seafile/scripts/utils/__init__.py @@ -0,0 +1,287 @@ +# coding: UTF-8 + +from __future__ import print_function +from ConfigParser import ConfigParser +from contextlib import contextmanager +import os +import datetime +from os.path import abspath, basename, exists, dirname, join, isdir, expanduser +import platform +import sys +import subprocess +import time +import logging +import logging.config +import click +import termcolor +import colorlog + +logger = logging.getLogger('.utils') + +DEBUG_ENABLED = os.environ.get('SEAFILE_DOCKER_VERBOSE', '').lower() in ('true', '1', 'yes') + +def eprint(*a, **kw): + kw['file'] = sys.stderr + print(*a, **kw) + +def identity(msg, *a, **kw): + return msg + +colored = identity if not os.isatty(sys.stdin.fileno()) else termcolor.colored +red = lambda s: colored(s, 'red') +green = lambda s: colored(s, 'green') + +def underlined(msg): + return '\x1b[4m{}\x1b[0m'.format(msg) + +def sudo(*a, **kw): + call('sudo ' + a[0], *a[1:], **kw) + +def _find_flag(args, *opts, **kw): + is_flag = kw.get('is_flag', False) + if is_flag: + return any([opt in args for opt in opts]) + else: + for opt in opts: + try: + return args[args.index(opt) + 1] + except ValueError: + pass + +def call(*a, **kw): + dry_run = kw.pop('dry_run', False) + quiet = kw.pop('quiet', DEBUG_ENABLED) + cwd = kw.get('cwd', os.getcwd()) + check_call = kw.pop('check_call', True) + reduct_args = kw.pop('reduct_args', []) + if not quiet: + toprint = a[0] + args = [x.strip('"') for x in a[0].split() if '=' not in x] + for arg in reduct_args: + value = _find_flag(args, arg) + toprint = toprint.replace(value, '{}**reducted**'.format(value[:3])) + logdbg('calling: ' + green(toprint)) + logdbg('cwd: ' + green(cwd)) + kw.setdefault('shell', True) + if not dry_run: + if check_call: + return subprocess.check_call(*a, **kw) + else: + return subprocess.Popen(*a, **kw).wait() + +@contextmanager +def cd(path): + path = expanduser(path) + olddir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(olddir) + +def must_makedir(p): + p = expanduser(p) + if not exists(p): + logger.info('created folder %s', p) + os.makedirs(p) + else: + logger.debug('folder %s already exists', p) + +def setup_colorlog(): + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + }, + 'colored': { + '()': 'colorlog.ColoredFormatter', + 'format': "%(log_color)s[%(asctime)s]%(reset)s %(blue)s%(message)s", + 'datefmt': '%m/%d/%Y %H:%M:%S', + }, + }, + 'handlers': { + 'default': { + 'level': 'INFO', + 'formatter': 'colored', + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + '': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': True + }, + 'django.request': { + 'handlers': ['default'], + 'level': 'WARN', + 'propagate': False + }, + } + }) + + logging.getLogger('requests.packages.urllib3.connectionpool').setLevel( + logging.WARNING) + + +def setup_logging(level=logging.INFO): + kw = { + 'format': '[%(asctime)s][%(module)s]: %(message)s', + 'datefmt': '%m/%d/%Y %H:%M:%S', + 'level': level, + 'stream': sys.stdout + } + + logging.basicConfig(**kw) + logging.getLogger('requests.packages.urllib3.connectionpool').setLevel( + logging.WARNING) + +def get_process_cmd(pid, env=False): + env = 'e' if env else '' + try: + return subprocess.check_output('ps {} -o command {}'.format(env, pid), + shell=True).strip().splitlines()[1] + # except Exception, e: + # print(e) + except: + return None + +def get_match_pids(pattern): + pgrep_output = subprocess.check_output( + 'pgrep -f "{}" || true'.format(pattern), + shell=True).strip() + return [int(pid) for pid in pgrep_output.splitlines()] + +def ask_for_confirm(msg): + confirm = click.prompt(msg, default='Y') + return confirm.lower() in ('y', 'yes') + +def confirm_command_to_run(cmd): + if ask_for_confirm('Run the command: {} ?'.format(green(cmd))): + call(cmd) + else: + sys.exit(1) + +def git_current_commit(): + return get_command_output('git rev-parse --short HEAD').strip() + +def get_command_output(cmd): + shell = not isinstance(cmd, list) + return subprocess.check_output(cmd, shell=shell) + +def ask_yes_or_no(msg, prompt='', default=None): + print('\n' + msg + '\n') + while True: + answer = raw_input(prompt + ' [yes/no] ').lower() + if not answer: + continue + + if answer not in ('yes', 'no', 'y', 'n'): + continue + + if answer in ('yes', 'y'): + return True + else: + return False + +def git_branch_exists(branch): + return call('git rev-parse --short --verify {}'.format(branch)) == 0 + +def to_unicode(s): + if isinstance(s, str): + return s.decode('utf-8') + else: + return s + +def to_utf8(s): + if isinstance(s, unicode): + return s.encode('utf-8') + else: + return s + +def git_commit_time(refspec): + return int(get_command_output('git log -1 --format="%ct" {}'.format( + refspec)).strip()) + +def get_seafile_version(): + return os.environ['SEAFILE_VERSION'] + +def get_install_dir(): + return join('/opt/seafile/' + get_conf('SEAFILE_SERVER', 'seafile-server') + '-{}'.format(get_seafile_version())) + +def get_script(script): + return join(get_install_dir(), script) + + +_config = None + +def get_conf(key, default=None): + key = key.upper() + return os.environ.get(key, default) + +def _add_default_context(context): + default_context = { + 'current_timestr': datetime.datetime.now().strftime('%m/%d/%Y %H:%M:%S'), + } + for k in default_context: + context.setdefault(k, default_context[k]) + +def render_template(template, target, context): + from jinja2 import Environment, FileSystemLoader + env = Environment(loader=FileSystemLoader(dirname(template))) + _add_default_context(context) + content = env.get_template(basename(template)).render(**context) + with open(target, 'w') as fp: + fp.write(content) + +def logdbg(msg): + if DEBUG_ENABLED: + msg = '[debug] ' + msg + loginfo(msg) + +def loginfo(msg): + msg = '[{}] {}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), green(msg)) + eprint(msg) + +def cert_has_valid_days(cert, days): + assert exists(cert) + + secs = 86400 * int(days) + retcode = call('openssl x509 -checkend {} -noout -in {}'.format(secs, cert), check_call=False) + return retcode == 0 + +def get_version_stamp_file(): + return '/shared/seafile/seafile-data/current_version' + +def read_version_stamp(fn=get_version_stamp_file()): + assert exists(fn), 'version stamp file {} does not exist!'.format(fn) + with open(fn, 'r') as fp: + return fp.read().strip() + +def update_version_stamp(version, fn=get_version_stamp_file()): + with open(fn, 'w') as fp: + fp.write(version + '\n') + +def wait_for_mysql(): + while not exists('/var/run/mysqld/mysqld.sock'): + logdbg('waiting for mysql server to be ready') + time.sleep(2) + logdbg('mysql server is ready') + +def wait_for_nginx(): + while True: + logdbg('waiting for nginx server to be ready') + output = get_command_output('netstat -nltp') + if ':80 ' in output: + logdbg(output) + logdbg('nginx is ready') + return + time.sleep(2) + +def replace_file_pattern(fn, pattern, replacement): + with open(fn, 'r') as fp: + content = fp.read() + with open(fn, 'w') as fp: + fp.write(content.replace(pattern, replacement)) diff --git a/cluster/image/pro_seafile/templates/letsencrypt.cron.template b/cluster/image/pro_seafile/templates/letsencrypt.cron.template new file mode 100644 index 0000000..cd877b6 --- /dev/null +++ b/cluster/image/pro_seafile/templates/letsencrypt.cron.template @@ -0,0 +1,3 @@ +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +# min hour dayofmonth month dayofweek command +0 0 1 * * root /scripts/ssl.sh {{ ssl_dir }} {{ domain }} diff --git a/cluster/image/pro_seafile/templates/seafile.nginx.conf.template b/cluster/image/pro_seafile/templates/seafile.nginx.conf.template new file mode 100644 index 0000000..fe02cb0 --- /dev/null +++ b/cluster/image/pro_seafile/templates/seafile.nginx.conf.template @@ -0,0 +1,81 @@ +# -*- mode: nginx -*- +# Auto generated at {{ current_timestr }} +{% if https -%} +server { + listen 80; + server_name _ default_server; + rewrite ^ https://{{ domain }}$request_uri? permanent; +} +{% endif -%} + +server { +{% if https -%} + listen 443; + ssl on; + ssl_certificate /shared/ssl/{{ domain }}.crt; + ssl_certificate_key /shared/ssl/{{ domain }}.key; + + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; + + # TODO: More SSL security hardening: ssl_session_tickets & ssl_dhparam + # ssl_session_tickets on; + # ssl_session_ticket_key /etc/nginx/sessionticket.key; + # ssl_session_cache shared:SSL:10m; + # ssl_session_timeout 10m; +{% else -%} + listen 80; +{% endif -%} + + server_name {{ domain }}; + + client_max_body_size 10m; + + location / { + proxy_pass http://127.0.0.1:8000/; + proxy_read_timeout 310s; + proxy_set_header Host $host; + proxy_set_header Forwarded "for=$remote_addr;proto=$scheme"; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Connection ""; + proxy_http_version 1.1; + } + + location /seafhttp { + rewrite ^/seafhttp(.*)$ $1 break; + proxy_pass http://127.0.0.1:8082; + client_max_body_size 0; + proxy_connect_timeout 36000s; + proxy_read_timeout 36000s; + } + + location /seafdav { + client_max_body_size 0; + fastcgi_pass 127.0.0.1:8080; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_script_name; + + fastcgi_param SERVER_PROTOCOL $server_protocol; + fastcgi_param QUERY_STRING $query_string; + fastcgi_param REQUEST_METHOD $request_method; + fastcgi_param CONTENT_TYPE $content_type; + fastcgi_param CONTENT_LENGTH $content_length; + fastcgi_param SERVER_ADDR $server_addr; + fastcgi_param SERVER_PORT $server_port; + fastcgi_param SERVER_NAME $server_name; + + access_log /var/log/nginx/seafdav.access.log; + error_log /var/log/nginx/seafdav.error.log; + } + + location /media { + root /opt/seafile/seafile-server-latest/seahub; + } + + # For letsencrypt + location /.well-known/acme-challenge/ { + alias /var/www/challenges/; + try_files $uri =404; + } +} diff --git a/cluster/scripts/gc.sh b/cluster/scripts/gc.sh index ac31fce..4531933 100755 --- a/cluster/scripts/gc.sh +++ b/cluster/scripts/gc.sh @@ -18,7 +18,7 @@ fi # Do it ( set +e - $SEAFILE_DIR/seaf-gc.sh | tee -a /var/log/gc.log + $SEAFILE_DIR/seaf-gc.sh "$@" | tee -a /var/log/gc.log # We want to presevent the exit code of seaf-gc.sh exit "${PIPESTATUS[0]}" ) diff --git a/image/Makefile b/image/Makefile index 8a71b4d..14e3ea7 100644 --- a/image/Makefile +++ b/image/Makefile @@ -1,9 +1,9 @@ server_version=6.3.7 -base_image=seafileltd/base:16.04 -base_image_squashed=seafileltd/base:16.04-squashed -pro_base_image=seafileltd/pro-base:16.04 -pro_base_image_squashed=seafileltd/pro-base:16.04-squashed +base_image=seafileltd/base:18.04 +base_image_squashed=seafileltd/base:18.04-squashed +pro_base_image=seafileltd/pro-base:18.04 +pro_base_image_squashed=seafileltd/pro-base:18.04-squashed server_image=seafileltd/seafile:$(server_version) server_image_squashed=seafileltd/seafile:$(server_version)-squashed pro_server_image=seafileltd/seafile-pro:$(server_version) @@ -17,9 +17,9 @@ all: @echo base: - docker pull phusion/baseimage:0.9.19 - docker-squash --tag phusion/baseimage:latest phusion/baseimage:0.9.19 - docker tag phusion/baseimage:latest phusion/baseimage:0.9.19 + docker pull phusion/baseimage:0.11 + docker-squash --tag phusion/baseimage:latest phusion/baseimage:0.11 + docker tag phusion/baseimage:latest phusion/baseimage:0.11 cd base && docker build -t $(base_image) . docker-squash --tag $(base_image_squashed) $(base_image) docker tag $(base_image_squashed) $(base_image) diff --git a/image/base/Dockerfile b/image/base/Dockerfile index a6e5729..e51c4fd 100644 --- a/image/base/Dockerfile +++ b/image/base/Dockerfile @@ -1,6 +1,6 @@ -# Lastet phusion baseimage as of 20180412, based on ubuntu 16.04 +# Lastet phusion baseimage as of 20180412, based on ubuntu 18.04 # See https://hub.docker.com/r/phusion/baseimage/tags/ -FROM phusion/baseimage:0.10.1 +FROM phusion/baseimage:0.11 ENV UPDATED_AT=20180412 \ DEBIAN_FRONTEND=noninteractive @@ -15,7 +15,7 @@ RUN apt-get install -qq -y vim htop net-tools psmisc git wget curl # Guidline for installing python libs: if a lib has C-compoment (e.g. # python-imaging depends on libjpeg/libpng), we install it use apt-get. # Otherwise we install it with pip. -RUN apt-get install -y python2.7-dev python-imaging python-ldap python-mysqldb +RUN apt-get install -y python2.7-dev python-ldap python-mysqldb zlib1g-dev libmemcached-dev gcc RUN curl -sSL -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ python /tmp/get-pip.py && \ rm -rf /tmp/get-pip.py && \ diff --git a/image/base/requirements.txt b/image/base/requirements.txt index 3094155..b8786ae 100644 --- a/image/base/requirements.txt +++ b/image/base/requirements.txt @@ -9,7 +9,8 @@ colorlog==2.7.0 Jinja2==2.8 MarkupSafe==0.23 # via jinja2 prettytable==0.7.2 -python-memcached==1.58 -six==1.10.0 # via python-memcached termcolor==1.1.0 urllib3==1.19 +Pillow==4.3.0 +pylibmc==1.6.0 +django-pylibmc==0.6.1 diff --git a/image/pro_base/Dockerfile b/image/pro_base/Dockerfile index 7f095f1..d5fe177 100644 --- a/image/pro_base/Dockerfile +++ b/image/pro_base/Dockerfile @@ -1,7 +1,12 @@ -FROM seafileltd/base:16.04 +FROM seafileltd/base:18.04 # syslog-ng and syslog-forwarder would mess up the container stdout, not good # when debugging/upgrading. + +# Fixing the "Sub-process /usr/bin/dpkg returned an error code (1)", +# when RUN apt-get +RUN mkdir -p /usr/share/man/man1 + RUN apt update RUN apt-get install -y openjdk-8-jre libmemcached-dev zlib1g-dev pwgen curl openssl poppler-utils libpython2.7 libreoffice \ @@ -9,7 +14,12 @@ libreoffice-script-provider-python ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy py RUN apt-get install -y tzdata python-pip python-setuptools python-urllib3 python-ldap python-ceph -RUN pip install pylibmc django-pylibmc boto twilio oss2 - +# The S3 storage, oss storage and psd online preview etc, +# depends on the python-backages as follow: +RUN pip install boto==2.43.0 \ + oss2==2.3.0 \ + psd-tools==1.4 \ + pycryptodome==3.7.2 \ + twilio==5.7.0 RUN apt clean diff --git a/image/pro_seafile/Dockerfile b/image/pro_seafile/Dockerfile index 83d0458..72f7a89 100644 --- a/image/pro_seafile/Dockerfile +++ b/image/pro_seafile/Dockerfile @@ -1,4 +1,4 @@ -FROM seafileltd/pro-base:16.04 +FROM seafileltd/pro-base:18.04 WORKDIR /opt/seafile ENV SEAFILE_VERSION=6.3.7 SEAFILE_SERVER=seafile-pro-server diff --git a/image/pro_seafile/templates/seafile.nginx.conf.template b/image/pro_seafile/templates/seafile.nginx.conf.template index fe02cb0..574f4ab 100644 --- a/image/pro_seafile/templates/seafile.nginx.conf.template +++ b/image/pro_seafile/templates/seafile.nginx.conf.template @@ -4,7 +4,16 @@ server { listen 80; server_name _ default_server; - rewrite ^ https://{{ domain }}$request_uri? permanent; + # allow certbot to connect to challenge location via HTTP Port 80 + # otherwise renewal request will fail + location /.well-known/acme-challenge/ { + alias /var/www/challenges/; + try_files $uri =404; + } + + location / { + rewrite ^ https://{{ domain }}$request_uri? permanent; + } } {% endif -%} diff --git a/image/seafile/Dockerfile b/image/seafile/Dockerfile index a0401ef..944c8a8 100644 --- a/image/seafile/Dockerfile +++ b/image/seafile/Dockerfile @@ -1,4 +1,4 @@ -FROM seafileltd/base:16.04 +FROM seafileltd/base:18.04 WORKDIR /opt/seafile RUN mkdir -p /etc/my_init.d diff --git a/image/seafile/templates/seafile.nginx.conf.template b/image/seafile/templates/seafile.nginx.conf.template index fe02cb0..30b04bd 100644 --- a/image/seafile/templates/seafile.nginx.conf.template +++ b/image/seafile/templates/seafile.nginx.conf.template @@ -4,7 +4,17 @@ server { listen 80; server_name _ default_server; - rewrite ^ https://{{ domain }}$request_uri? permanent; + + # allow certbot to connect to challenge location via HTTP Port 80 + # otherwise renewal request will fail + location /.well-known/acme-challenge/ { + alias /var/www/challenges/; + try_files $uri =404; + } + + location / { + rewrite ^ https://{{ domain }}$request_uri? permanent; + } } {% endif -%} diff --git a/scripts/gc.sh b/scripts/gc.sh index ac31fce..4531933 100755 --- a/scripts/gc.sh +++ b/scripts/gc.sh @@ -18,7 +18,7 @@ fi # Do it ( set +e - $SEAFILE_DIR/seaf-gc.sh | tee -a /var/log/gc.log + $SEAFILE_DIR/seaf-gc.sh "$@" | tee -a /var/log/gc.log # We want to presevent the exit code of seaf-gc.sh exit "${PIPESTATUS[0]}" )