From 7814d43b12adbe7c2ab34c15c1f43614a5fce4df Mon Sep 17 00:00:00 2001 From: Shuai Lin Date: Fri, 11 Nov 2016 12:54:47 +0800 Subject: [PATCH] Initial work on running seafile with docker. --- .gitignore | 3 + .travis.yml | 2 +- containers/.gitkeep | 0 image/Makefile | 9 ++ image/base/Dockerfile | 16 +++ image/base/memcached.sh | 4 + image/seafile/Dockerfile | 13 ++ image/seafile/create_data_links.sh | 24 ++++ launcher | 78 +++++++++++ scripts/bootstrap.py | 47 +++++++ scripts/config.py | 8 ++ scripts/start.py | 0 scripts/utils/__init__.py | 206 +++++++++++++++++++++++++++++ seafile-server-setup | 107 +++++++++++++++ shared/logs/.gitkeep | 0 15 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 containers/.gitkeep create mode 100644 image/Makefile create mode 100644 image/base/Dockerfile create mode 100755 image/base/memcached.sh create mode 100644 image/seafile/Dockerfile create mode 100755 image/seafile/create_data_links.sh create mode 100755 launcher create mode 100755 scripts/bootstrap.py create mode 100644 scripts/config.py create mode 100644 scripts/start.py create mode 100644 scripts/utils/__init__.py create mode 100755 seafile-server-setup create mode 100644 shared/logs/.gitkeep diff --git a/.gitignore b/.gitignore index f54cb69..ba7756a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ *.swp .DS_Store *.pyc + +containers/ +shared/ diff --git a/.travis.yml b/.travis.yml index a85d9e9..0471aa5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,4 @@ install: - echo "Nothing to install" script: - - cd base && docker build -t seafile/base . + - cd image && make base && make diff --git a/containers/.gitkeep b/containers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/image/Makefile b/image/Makefile new file mode 100644 index 0000000..1b76ae0 --- /dev/null +++ b/image/Makefile @@ -0,0 +1,9 @@ +version=6.0.5 + +all: + cd seafile && docker build -t seafileorg/server:$(version) . + +base: + cd base && docker build -t seafileorg/base:16.04 . + +.PHONY: base diff --git a/image/base/Dockerfile b/image/base/Dockerfile new file mode 100644 index 0000000..019327a --- /dev/null +++ b/image/base/Dockerfile @@ -0,0 +1,16 @@ +# lastet phusion baseimage as of 2016.11, based on ubuntu 16.04 +FROM phusion/baseimage:0.9.19 + +ENV UPDATED_AT 20161110 + +RUN apt-get update -qq && apt-get -qq -y install python2.7-dev memcached python-pip \ + python-setuptools python-imaging python-mysqldb python-memcache python-ldap \ + python-urllib3 sqlite3 nginx \ + vim htop net-tools psmisc git wget curl + +RUN pip install -U wheel && pip install click termcolor prettytable colorlog + +RUN mkdir /etc/service/memcached +ADD memcached.sh /etc/service/memcached/run + +CMD ["/sbin/my_init", "--", "bash", "-l"] diff --git a/image/base/memcached.sh b/image/base/memcached.sh new file mode 100755 index 0000000..2a50a10 --- /dev/null +++ b/image/base/memcached.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# `/sbin/setuser memcache` runs the given command as the user `memcache`. +# If you omit that part, the command will be run as root. +exec /sbin/setuser memcache /usr/bin/memcached >>/var/log/memcached.log 2>&1 diff --git a/image/seafile/Dockerfile b/image/seafile/Dockerfile new file mode 100644 index 0000000..2289380 --- /dev/null +++ b/image/seafile/Dockerfile @@ -0,0 +1,13 @@ +FROM seafileorg/base:16.04 +WORKDIR /opt/seafile + +ENV SEAFILE_VERSION=6.0.5 + +RUN mkdir -p /opt/seafile/ && \ + curl -sSL -o - https://bintray.com/artifact/download/seafile-org/seafile/seafile-server_6.0.5_x86-64.tar.gz \ + | tar xzf - -C /opt/seafile/ + +RUN mkdir -p /etc/my_init.d +ADD create_data_links.sh /etc/my_init.d/create_data_links.sh + +CMD ["/sbin/my_init", "--", "bash", "-l"] diff --git a/image/seafile/create_data_links.sh b/image/seafile/create_data_links.sh new file mode 100755 index 0000000..be349b8 --- /dev/null +++ b/image/seafile/create_data_links.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e +set -o pipefail + +dirs=( + conf + ccnet + logs + seafile-data + seahub-data + seahub.db +) + +for d in ${dirs[*]}; do + src=/shared/$d + if [[ -e $src ]]; then + ln -sf $src /opt/seafile/ + fi +done + +ln -sf /opt/seafile/seafile-server-${SEAFILE_VERSION} /opt/seafile/seafile-server-latest + +# TODO: create avatars link diff --git a/launcher b/launcher new file mode 100755 index 0000000..213ac58 --- /dev/null +++ b/launcher @@ -0,0 +1,78 @@ +#!/bin/bash + +set -e +set -o pipefail + +version=6.0.5 +image=seafileorg/server:$version +topdir=$(cd $(dirname $0); pwd -P) +sharedir=$topdir/shared + +cd $topdir + +dbg() { + if [[ $debug == "true" ]]; then + echo "dbg: $1" + fi +} + +err_and_quit () { + printf "\n\n\033[33mError: %s\033[m\n\n" "$1" + exit 1 +} + +set_volumes() { + local mounts seahub_db + + seahub_db=$sharedir/seahub.db + if [[ ! -e $seahub_db ]]; then + touch $seahub_db + fi + + local bash_history=$sharedir/.bash_history + if [[ ! -e $bash_history ]]; then + touch $bash_history + fi + + mounts=( + $sharedir:/shared + $topdir/containers:/containers:ro + $topdir/scripts:/scripts:ro + $bash_history:/root/.bash_history + ) + volumes="" + local m + for m in ${mounts[*]}; do + volumes="$volumes -v $m" + done +} + +bootstrap() { + set_volumes + docker run --rm -it $volumes $image /scripts/bootstrap.py +} + +start() { + set_volumes + docker run --rm -it $volumes $image # scripts/start.py +} + +enter() { + err_and_quit "Not implemented yet" +} + +function main { + local action + while [[ $# -gt 0 ]] + do + case "$1" in + bootstrap|start|enter) action=$1 ; shift 1 ;; + --debug) debug=true ; shift 1 ;; + --dummy) dummy=$2 ; shift 2 ;; + *) err_and_quit "Argument error. Please see help." ;; + esac + done + "$action" +} + +main "$@" diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100755 index 0000000..8f2faa2 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +#coding: UTF-8 + +""" +This script calls the appropriate seafile init scripts (e.g. +setup-seafile.sh or setup-seafile-mysql.sh. It's supposed to run inside the +container. +""" + +from ConfigParser import ConfigParser +import os +from os.path import abspath, basename, exists, dirname, join, isdir +import shutil +import sys + +from utils import call, get_install_dir, get_script + +installdir = get_install_dir() +topdir = dirname(installdir) + +_config = None + +def get_conf(key): + global _config + if _config is None: + _config = ConfigParser() + _config.read("/containers/bootstrap.conf") + return _config.get("server", key) + +def main(): + env = { + 'SERVER_NAME': 'seafile', + 'SERVER_IP': get_conf("server.hostname"), + } + call('{} auto'.format(get_script('setup-seafile.sh')), env=env) + for fn in ('conf', 'ccnet', 'seafile-data', 'seahub-data', 'seahub.db'): + src = join(topdir, fn) + dst = join('/shared', fn) + if exists(dst): + if isdir(dst): + shutil.rmtree(dst) + else: + os.unlink(dst) + shutil.move(src, '/shared') + +if __name__ == '__main__': + main() diff --git a/scripts/config.py b/scripts/config.py new file mode 100644 index 0000000..f1b88b4 --- /dev/null +++ b/scripts/config.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +#coding: UTF-8 + +def main(): + pass + +if __name__ == '__main__': + main() diff --git a/scripts/start.py b/scripts/start.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py new file mode 100644 index 0000000..cc65fe2 --- /dev/null +++ b/scripts/utils/__init__.py @@ -0,0 +1,206 @@ +# coding: UTF-8 + +from __future__ import print_function +import os +import platform +import sys +import termcolor +import subprocess +import logging +import logging.config +import click +import colorlog +from os.path import abspath, basename, exists, dirname, join, isdir, expanduser +from contextlib import contextmanager + +logger = logging.getLogger('.utils') + +def eprint(*a, **kw): + kw['file'] = sys.stderr + print(*a, **kw) + +def identity(msg, *a, **kw): + return msg + +colored = identity if not os.isatty(sys.stdin.fileno()) else termcolor.colored +red = lambda s: colored(s, 'red') +green = lambda s: colored(s, 'green') + +def underlined(msg): + return '\x1b[4m{}\x1b[0m'.format(msg) + +def sudo(*a, **kw): + call('sudo ' + a[0], *a[1:], **kw) + +def _find_flag(args, *opts, **kw): + is_flag = kw.get('is_flag', False) + if is_flag: + return any([opt in args for opt in opts]) + else: + for opt in opts: + try: + return args[args.index(opt) + 1] + except ValueError: + pass + +def call(*a, **kw): + dry_run = kw.pop('dry_run', False) + quiet = kw.pop('quiet', False) + cwd = kw.get('cwd', os.getcwd()) + reduct_args = kw.pop('reduct_args', []) + if not quiet: + toprint = a[0] + args = [x.strip('"') for x in a[0].split() if '=' not in x] + for arg in reduct_args: + value = _find_flag(args, arg) + toprint = toprint.replace(value, '{}**reducted**'.format(value[:3])) + eprint('calling: ', green(toprint)) + eprint('cwd: ', green(cwd)) + kw.setdefault('shell', True) + if not dry_run: + return subprocess.Popen(*a, **kw).wait() + +@contextmanager +def cd(path): + path = expanduser(path) + olddir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(olddir) + +def must_makedir(p): + p = expanduser(p) + if not exists(p): + logger.info('created folder %s', p) + os.makedirs(p) + else: + logger.debug('folder %s already exists', p) + +def setup_colorlog(): + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + }, + 'colored': { + '()': 'colorlog.ColoredFormatter', + 'format': "%(log_color)s[%(asctime)s]%(reset)s %(blue)s%(message)s", + 'datefmt': '%m/%d/%Y %H:%M:%S', + }, + }, + 'handlers': { + 'default': { + 'level': 'INFO', + 'formatter': 'colored', + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + '': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': True + }, + 'django.request': { + 'handlers': ['default'], + 'level': 'WARN', + 'propagate': False + }, + } + }) + + logging.getLogger('requests.packages.urllib3.connectionpool').setLevel( + logging.WARNING) + + +def setup_logging(level=logging.INFO): + kw = { + 'format': '[%(asctime)s][%(module)s]: %(message)s', + 'datefmt': '%m/%d/%Y %H:%M:%S', + 'level': level, + 'stream': sys.stdout + } + + logging.basicConfig(**kw) + logging.getLogger('requests.packages.urllib3.connectionpool').setLevel( + logging.WARNING) + +def get_process_cmd(pid, env=False): + env = 'e' if env else '' + try: + return subprocess.check_output('ps {} -o command {}'.format(env, pid), + shell=True).strip().splitlines()[1] + # except Exception, e: + # print(e) + except: + return None + +def get_match_pids(pattern): + pgrep_output = subprocess.check_output( + 'pgrep -f "{}" || true'.format(pattern), + shell=True).strip() + return [int(pid) for pid in pgrep_output.splitlines()] + +def ask_for_confirm(msg): + confirm = click.prompt(msg, default='Y') + return confirm.lower() in ('y', 'yes') + +def confirm_command_to_run(cmd): + if ask_for_confirm('Run the command: {} ?'.format(green(cmd))): + call(cmd) + else: + sys.exit(1) + +def git_current_commit(): + return get_command_output('git rev-parse --short HEAD').strip() + +def get_command_output(cmd): + shell = not isinstance(cmd, list) + return subprocess.check_output(cmd, shell=shell) + +def ask_yes_or_no(msg, prompt='', default=None): + print('\n' + msg + '\n') + while True: + answer = raw_input(prompt + ' [yes/no] ').lower() + if not answer: + continue + + if answer not in ('yes', 'no', 'y', 'n'): + continue + + if answer in ('yes', 'y'): + return True + else: + return False + +def git_branch_exists(branch): + return call('git rev-parse --short --verify {}'.format(branch)) == 0 + +def to_unicode(s): + if isinstance(s, str): + return s.decode('utf-8') + else: + return s + +def to_utf8(s): + if isinstance(s, unicode): + return s.encode('utf-8') + else: + return s + +def git_commit_time(refspec): + return int(get_command_output('git log -1 --format="%ct" {}'.format( + refspec)).strip()) + +def get_seafile_version(): + return os.environ['SEAFILE_VERSION'] + +def get_install_dir(): + return join('/opt/seafile/seafile-server-{}'.format(get_seafile_version())) + +def get_script(script): + return join(get_install_dir(), script) diff --git a/seafile-server-setup b/seafile-server-setup new file mode 100755 index 0000000..dca8ce7 --- /dev/null +++ b/seafile-server-setup @@ -0,0 +1,107 @@ +#!/bin/bash + +set -e +set -o pipefail + +err_and_quit () { + printf "\n\n\033[33mError occured during setup. \nPlease fix possible issues and run the script again.\033[m\n\n" + exit 1 +} + +on_ctrl_c_pressed () { + printf "\n\n\033[33mYou have pressed Ctrl-C. Setup is interrupted.\033[m\n\n" + exit 1 +} + +# clean newly created ccnet/seafile configs when exit on SIGINT +trap on_ctrl_c_pressed 2 + +read_yes_no () { + printf "[yes|no] " + read yesno + while [[ "$yesno" != "yes" && "$yesno" != "no" ]] + do + printf "please answer [yes|no] " + read yesno + done + + if [[ "$yesno" == "no" ]]; then + return 1 + else + return 0 + fi +} + +ask_question () { + local question default key + question=$1 + default=$2 + key=$3 + printf "$question" + printf "\n" + if [[ "$default" != "" && "$default" != "nodefault" ]]; then + printf "[default: $default] " + elif [[ "$key" != "" ]]; then + printf "[$key]: " + fi +} + +get_server_name () { + local question="Host name for your seafile server?" default="seafile.example.com" + ask_question "$question" "seafile.example.com" + read server_name + if [[ "$server_name" == "" ]]; then + server_name=$default + elif [[ ! $server_name =~ ^[a-zA-Z0-9_-.]+$ ]]; then + printf "\n\033[33m${server_name}\033[m is not a valid name.\n" + get_server_name + fi + echo +} + +# echo "Please specify the email address and password for the seahub administrator." +# echo "You can use them to login as admin on your seahub website." +# echo + +get_admin_email () { + local question="Admin email address for your seafile server?" default="me@example.com" + ask_question "$question" "$default" + read admin_email + if [[ "$admin_email" == "" ]]; then + admin_email=$default + elif [[ ! $admin_email =~ ^.+@.*\..+$ ]]; then + echo "$admin_email is not a valid email address" + get_admin_email + fi +} + +get_admin_passwd () { + local question="Admin password for your seafile server?" + ask_question "$question" "nodefault" "seahub admin password" + read -s admin_passwd + echo + question="Please enter the password again:" + ask_question "$question" "nodefault" "seahub admin password again" + read -s admin_passwd_again + echo + if [[ "$admin_passwd" != "$admin_passwd_again" ]]; then + printf "\033[33mThe passwords didn't match.\033[m" + get_admin_passwd + elif [[ "$admin_passwd" == "" ]]; then + echo "Password cannot be empty." + get_admin_passwd + fi +} + +get_server_name +get_admin_email +get_admin_passwd + +cat >containers/bootstrap.conf<