misc/docker/synology: package Perkeep for Synology

Fixes perkeep#986

Change-Id: I63b506944e3d71a3e864d9f4b37a1e0b1abd8b9c

doc/synology.md

@@ -0,0 +1,29 @@
+# Perkeep on Synology appliances
+
+## Installation
+
+To facilitate running Perkeep on Synology appliances, we try to provide packages (`.spk` files) that can be readily installed through the **Package Center**. They have been built for **DSM 6.2** but they should at least work for **DSM 6.1** as well. For now the packages have not been published in the Synology Package Center "market", so you need to download them manually among the ones we host:
+
+* [Perkeep-6281-latest.spk](https://storage.googleapis.com/perkeep-release/synology/Perkeep-armv5-b5c76a70e8c8d0b158a3dd19c261c0b2e62cd693.spk)
+* [Perkeep-x64-latest.spk](https://storage.googleapis.com/perkeep-release/synology/Perkeep-x86_64-b5c76a70e8c8d0b158a3dd19c261c0b2e62cd693.spk)
+
+Before installing, as we use the admin user home directory to store data, you need to make sure user homes are enabled. To do that: go to **Control Panel** -> **User** -> **Advanced** tab -> **User Home**, and tick **Enable user home service**.
+
+As you have downloaded the package yourself, it has to be installed manually as well, which means clicking on the **Manual Install** button in the Package Center. Then follow the installation instruction.
+
+After the installation, the Perkeep server (perkeepd) can be started and stopped from the package page in the **Package Center**.
+
+## Configuration
+
+As a Synology comes with an HTTP server (nginx) already listening on standard ports (80 and 443), making Perkeep accessible on these ports requires configuring nginx accordingly, which we do not do for now. One of the simplest ways to do so, is to leave Perkeep listening on HTTP on a non-privileged port (like 3179, in the default configuration), and to add a reverse proxy to the nginx configuration: go to **Control Panel** -> **Application Portal** -> **Reverse Proxy** tab. For example, if your NAS domain is **example.com**, and you control the subdomains for it, you can make the source be **perkeep.example.com** (HTTPS, port 443), and the destination be **localhost** (HTTP, port 3179).
+
+For the same reason (not supported by default nginx configuration), websockets will not be working in Perkeep.
+All of the nginx configuration is in `/etc/nginx`, but it is out of the scope of this document to explain it.
+
+The Perkeep configuration file is located at `/var/services/homes/admin/.config/perkeep/server-config.json`. You can use the **File Station** to download it, edit it, and upload it again to suit your needs. Or you can login with ssh and then use a console text editor to modify it.
+
+The data (blobs and index) are stored by default at `/var/services/homes/admin/var/perkeep`.
+
+The logs are at `/var/services/homes/admin/var/perkeep.log`.
+
+The script controlling the service is installed at `/var/packages/Perkeep/scripts/start-stop-status`. For easier debugging, you can use it to manually stop and start the server.

misc/docker/synology/Dockerfile

@@ -0,0 +1,71 @@
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+FROM buildpack-deps:stretch-scm AS pkbuild
+
+MAINTAINER Perkeep Authors <[email protected]>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# gcc for cgo, sqlite
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		g++ \
+		gcc \
+		libc6-dev \
+		make \
+		pkg-config \
+        libsqlite3-dev
+
+ENV GOLANG_VERSION 1.10.2
+ARG perkeep_version=8b537a66307cf41a659786f1a898c77b46303601
+
+WORKDIR /usr/local
+RUN wget -O go.tgz https://dl.google.com/go/go$GOLANG_VERSION.linux-amd64.tar.gz
+RUN echo "4b677d698c65370afa33757b6954ade60347aaca310ea92a63ed717d7cb0c2ff go.tgz" | sha256sum -c -
+RUN tar -zxvf go.tgz
+
+ENV GOROOT /usr/local/go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+WORKDIR $GOPATH/src
+RUN git clone https://perkeep.googlesource.com/perkeep perkeep.org
+WORKDIR $GOPATH/src/perkeep.org
+RUN git reset --hard $perkeep_version
+
+ARG goarch=amd64
+RUN go run make.go -v -arch=$goarch -arm=5
+
+#stage 2
+
+FROM ubuntu:16.04
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get -y upgrade
+RUN apt-get -y --no-install-recommends install ca-certificates git python3 xz-utils udev vim
+
+RUN mkdir -p /toolkit
+WORKDIR /toolkit
+RUN git clone https://github.com/SynologyOpenSource/pkgscripts-ng pkgscripts
+WORKDIR /toolkit/pkgscripts
+RUN git reset --hard 86409bbab301428b893bc3d099ba8ba29f22137d
+
+ARG dsm=6.2
+ARG arch=x64
+ENV BUILD_ENV ds.$arch-$dsm
+RUN echo "Preparing to build for: $BUILD_ENV"
+RUN ./EnvDeploy -v $dsm -p $arch
+WORKDIR /toolkit/build_env/$BUILD_ENV
+
+WORKDIR /toolkit
+ARG perkeep_version=8b537a66307cf41a659786f1a898c77b46303601
+RUN mkdir -p source
+WORKDIR /toolkit/source
+ADD perkeep perkeep
+RUN sed -i s:version=SET_BY_DOCKER_BUILD:version=\"$perkeep_version\": perkeep/INFO.sh
+
+ARG gobin=/go/bin
+COPY --from=pkbuild $gobin/pk* /toolkit/build_env/$BUILD_ENV/root/bin/
+COPY --from=pkbuild $gobin/perkeepd /toolkit/build_env/$BUILD_ENV/root/bin/
+
+WORKDIR /toolkit

misc/docker/synology/build_syno.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2018 The Perkeep Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Command build_syno builds and packages Perkeep for Synology appliances.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+var (
+	// TODO(mpl): include more arches.
+	flagArch       = flag.String("arch", "x64", "Synology architecture to build for. Possible values are limited to x64 or 6281 for now.")
+	flagDsm        = flag.String("dsm", "6.2", "DSM version to build for.")
+	flagPerkeepRev = flag.String("pkrev", "8b537a66307cf41a659786f1a898c77b46303601", "git revision of Perkeep to package")
+	flagNoCache    = flag.Bool("nocache", false, "build docker image with --no-cache")
+)
+
+var pwd string
+
+func main() {
+	flag.Parse()
+	if *flagArch != "x64" && *flagArch != "6281" {
+		log.Fatalf("unsupported architecture: %v", *flagArch)
+	}
+
+	newCwd := filepath.Dir(flag.Arg(0))
+	if err := os.Chdir(newCwd); err != nil {
+		log.Fatalf("error changing dir to %v: %v", newCwd, err)
+	}
+	var err error
+	pwd, err = os.Getwd()
+	if err != nil {
+		log.Fatalf("error getting current directory: %v", err)
+	}
+
+	gobin := "/go/bin"
+	goarch := "amd64"
+	// TODO(mpl): figure out the correspondance between all the other arches and the
+	// values for the go vars.
+	if *flagArch == "6281" {
+		gobin = "/go/bin/linux_arm"
+		goarch = "arm"
+	}
+
+	// docker build -t perkeep/synology --build-arg arch=x64 --build-arg dsm=6.2 --build-arg gobin=/go/bin --build-arg goarch=amd64
+	args := []string{"build"}
+	if *flagNoCache {
+		args = append(args, "--no-cache")
+	}
+	args = append(args, "-t", "perkeep/synology",
+		"--build-arg", "arch="+*flagArch,
+		"--build-arg", "dsm="+*flagDsm,
+		"--build-arg", "gobin="+gobin,
+		"--build-arg", "goarch="+goarch,
+		"--build-arg", "perkeep_version="+*flagPerkeepRev,
+		".")
+	cmd := exec.Command("docker", args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatalf("Error building synology image: %v, %s", err, out)
+	}
+	fmt.Println(string(out))
+
+	if err := os.MkdirAll(filepath.Join(pwd+"/out"), 0755); err != nil {
+		log.Fatalf("Error creating out dir: %v", err)
+	}
+
+	// the actual building step (./pkgscripts/PkgCreate.py) can't be in the
+	// Dockerfile since it's doing higher privilege stuff like chroot and mounts.
+	// Which is why we run it as the last step, and in privileged mode.
+
+	// TODO(mpl): we might be able to "just" run a furtherly modified version of
+	// perkeep/SynoBuildConf/install because at the end of the day, we've already built the
+	// binaries, and all we need to do is package them in an .spk. But that would require
+	// further understanding and rewriting of the various pieces involved, and I don't think
+	// it's worth spending time on it for now.
+	// Although if we manage that, another win is that then we can skip EnvDeploy,
+	// which means avoiding a heavy download.
+
+	// For runPkgCreate, you need to fetch the GPG keys at
+	// https://drive.google.com/drive/folders/1P95Lk1U6nA6kaVaxKRPi4Dv2Het-CWlY?usp=sharing
+	// and add them to $HOME/keys/perkeep-synology , which is then mounted to the container.
+	runPkgCreate()
+}
+
+// runPkgCreate runs the actual build/install step provided in the DSM toolkit: ./pkgscripts/PkgCreate.py
+// It can't be run during the build stage (in the Dockerfile), because it does
+// privileged operations (at least chroot) so it is run as a privileged container.
+// It requires the GPG keys at
+// https://drive.google.com/drive/folders/1P95Lk1U6nA6kaVaxKRPi4Dv2Het-CWlY?usp=sharing
+func runPkgCreate() {
+	// To test the equivalent from the shell:
+	// docker run --rm -i -t --privileged -v $PWD/out:/toolkit/result_spk -v $HOME/keys/perkeep-synology:/toolkit/build_env/ds.6281-6.2/root/.gnupg perkeep/synology ./pkgscripts/PkgCreate.py -v 6.2 -p x64 -x0 -c --print-log --build-opt="-J" perkeep
+	build_env := "ds." + *flagArch + "-" + *flagDsm
+	// TODO(mpl): test discrete --cap-add flags when I have a docker that supports them.
+	cmd := exec.Command("docker", "run", "--rm", "--privileged",
+		"-v", filepath.Join(pwd+"/out")+":/toolkit/result_spk",
+		"-v", filepath.Join(os.Getenv("HOME"), "keys", "perkeep-synology")+":/toolkit/build_env/"+build_env+"/root/.gnupg",
+		"perkeep/synology",
+		"./pkgscripts/PkgCreate.py", "-p", *flagArch, "-v", *flagDsm, "-x0", "-c", "--print-log", `--build-opt="-J"`, "perkeep")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatalf("Error building synology package: %v, %s", err, out)
+	}
+	fmt.Println(string(out))
+}

misc/docker/synology/perkeep/INFO.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright (c) 2000-2016 Synology Inc. All rights reserved.
+
+source /pkgscripts/include/pkg_util.sh
+
+package="Perkeep"
+version=SET_BY_DOCKER_BUILD
+displayname="Perkeep"
+maintainer="Perkeep Authors <[email protected]>"
+arch="$(pkg_get_unified_platform)"
+description="Perkeep lets you permanently keep your stuff, for life. See https://perkeep.org/doc/synology"
+helpurl="https://perkeep.org/doc/synology"
+[ "$(caller)" != "0 NULL" ] && return 0
+pkg_dump_info

misc/docker/synology/perkeep/SynoBuildConf/build

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+# TOOD(mpl): remove if not needed
+
+exit 0

misc/docker/synology/perkeep/SynoBuildConf/depends

@@ -0,0 +1,6 @@
+# Copyright (C) 2000-2016 Synology Inc. All rights reserved.
+
+# Keeping this file and section because they're in the doc and examples, but
+# this is overriden by the -v flag of PkgCreate.py anyway
+[default]
+all="6.2"

misc/docker/synology/perkeep/SynoBuildConf/install

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (C) 2000-2016 Synology Inc. All rights reserved.
+
+### Use PKG_DIR as working directory.
+PKG_DIR=/tmp/_test_spk
+rm -rf $PKG_DIR
+mkdir -p $PKG_DIR
+
+### get spk packing functions
+source /pkgscripts/include/pkg_util.sh
+
+create_package_tgz() {
+	local firewere_version=
+	local package_tgz_dir=/tmp/_package_tgz
+	local binary_dir=$package_tgz_dir/usr/bin
+
+	### clear destination directory
+	rm -rf $package_tgz_dir && mkdir -p $package_tgz_dir
+
+	### install needed file into PKG_DIR
+	mkdir -p $binary_dir
+	cp -av /root/bin/pk* /root/bin/perkeepd $binary_dir
+
+	### create package.tgz $1: source_dir $2: dest_dir
+	pkg_make_package $package_tgz_dir "${PKG_DIR}"
+}
+
+create_spk(){
+	local scripts_dir=$PKG_DIR/scripts
+
+	### Copy package center scripts to PKG_DIR
+	mkdir -p $scripts_dir
+	cp -av scripts/* $scripts_dir
+
+	### Copy package icon
+	cp -av PACKAGE_ICON*.PNG $PKG_DIR
+
+	### Generate INFO file
+	./INFO.sh > INFO
+	cp INFO $PKG_DIR/INFO
+
+	### Create the final spk.
+	# pkg_make_spk <source path> <dest path> <spk file name>
+	# Please put the result spk into /image/packages
+	# spk name functions: pkg_get_spk_name pkg_get_spk_unified_name pkg_get_spk_family_name
+	mkdir -p /image/packages
+	pkg_make_spk ${PKG_DIR} "/image/packages" $(pkg_get_spk_family_name)
+}
+
+main() {
+	create_package_tgz
+	create_spk
+}
+
+main "$@"

misc/docker/synology/perkeep/scripts/postinst

@@ -0,0 +1,30 @@
+#!/bin/sh
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+### This script will be executed when package installed and upgraded.
+### Actions after package installed.
+### ex. create database, create symbolic link...
+
+# make sure it's only run during package installation
+if test -z "$SYNOPKG_PKGDEST"
+then
+	exit 1
+fi
+
+# according to the doc, the owner of the files is the user that will be used to
+# run the service (even though that does not seem to be enough/working).
+# user http should always exist, so we need not bother with creating a perkeep user.
+chown http:http $SYNOPKG_PKGDEST/usr/bin/*
+ln -sf $SYNOPKG_PKGDEST/usr/bin/perkeepd /usr/bin/
+ln -sf $SYNOPKG_PKGDEST/usr/bin/pk /usr/bin/
+ln -sf $SYNOPKG_PKGDEST/usr/bin/pk-get /usr/bin/
+ln -sf $SYNOPKG_PKGDEST/usr/bin/pk-put /usr/bin/
+ln -sf $SYNOPKG_PKGDEST/usr/bin/pk-mount /usr/bin/
+# we persist the conf, blobs, and log in the admin home.
+# then users can view/modify the conf themselves in the file station.
+mkdir -p /var/services/homes/admin/var
+
+echo "Please see https://perkeep.org/doc/synology for the next steps." > $SYNOPKG_TEMP_LOGFILE
+
+exit 0

misc/docker/synology/perkeep/scripts/postuninst

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+### This script will be executed when package uninstalled and upgraded.
+### Actions after package uninstalled.
+### ex. remove garbage files.
+
+cd /usr/bin
+rm -f perkeepd pk pk-get pk-put pk-mount
+
+exit 0

misc/docker/synology/perkeep/scripts/postupgrade

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Copyright (C) 2000-2016 Synology Inc. All rights reserved.
+
+### This script will be executed ONLY at package upgraded.
+### Actions after package upgraded.
+### ex. restore user settings.
+
+exit 0

misc/docker/synology/perkeep/scripts/preinst

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+### This script will be execute when package installed and upgraded.
+### Actions before package installed.
+### ex. check environment.
+
+exit 0

misc/docker/synology/perkeep/scripts/preuninst

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+### This script will be executed when package uninstalled and upgraded.
+### Actions before package uninstalled
+### ex. backup package data.
+
+exit 0

misc/docker/synology/perkeep/scripts/preupgrade

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Copyright 2018 The Perkeep Authors.
+# Licensed under the Apache License, Version 2.0
+
+### This script will execute ONLY when package upgraded.
+### Actions before package upgraded.
+### ex. backup user settings for package upgrade.
+
+# TODO(mpl): figure out if anything to do for upgrades
+
+exit 0