Create new algorand-devtools package (algorand#1492)

installer/debian/algorand-devtools/conffiles

@@ -0,0 +1,2 @@
+/etc/apt/apt.conf.d/53algorand-devtools-upgrades
+

installer/debian/algorand-devtools/control

@@ -0,0 +1,10 @@
+Package: @PKG_NAME@
+Homepage: https://www.algorand.com/
+Maintainer: Algorand developers <[email protected]>
+Version: @VER@
+Architecture: @ARCH@
+Pre-Depends: algorand (>= @VER@)
+Recommends: unattended-upgrades
+Description: Algorand tools software
+ This package provides development tools for the Algorand blockchain.
+

installer/debian/preinst → installer/debian/algorand/preinst

@@ -13,7 +13,9 @@ if dpkg-query --list 'algorand*' &> /dev/null
 then
     if PKG_INFO=$(dpkg-query --show --showformat='${Package} ${Status}\n' 'algorand*' | grep "install ok installed")
     then
-        INSTALLED_PKG=$(grep -v algorand-indexer <<< "$PKG_INFO" | awk '{print $1}')
+        # Filter out `algorand-indexer` and `algorand-devtools` packages, they are allowed to be
+        # installed alongside other `algorand` packages.
+        INSTALLED_PKG=$(grep -v -e algorand-indexer -e algorand-devtools <<< "$PKG_INFO" | awk '{print $1}')
 
         if [ -n "$INSTALLED_PKG" ]
         then

installer/rpm/algorand-devtools/algorand-devtools.repo

@@ -0,0 +1,6 @@
+[algorand]
+name=Algorand-devtools
+baseurl=https://releases.algorand.com/rpm/stable/
+enabled=1
+gpgcheck=1
+gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub

installer/rpm/algorand-devtools/algorand-devtools.spec

@@ -0,0 +1,46 @@
+Name:          @PKG_NAME@
+Version:       @VER@
+Release:       1
+Summary:       Algorand tools software
+URL:           https://www.algorand.com
+License:       AGPL-3+
+Requires:      algorand >= @VER@
+
+%define SRCDIR go-algorand-rpmbuild
+%define _buildshell /bin/bash
+
+%description
+This package provides development tools for the Algorand blockchain.
+
+%license
+%include %{LICENSE_FILE}
+
+%prep
+## Nothing to prep; intended to be built using scripts/release/mule/package/{OS_TYPE}/{ARCH}/rpm/package.sh
+
+%build
+## Nothing to prep; intended to be built using scripts/release/mule/package/{OS_TYPE}/{ARCH}/rpm/package.sh
+
+%install
+mkdir -p %{buildroot}/usr/bin
+# NOTE: keep in sync with scripts/build_deb.sh bin_files
+# NOTE: keep in sync with %files section below
+for f in carpenter catchupsrv msgpacktool tealcut tealdbg; do
+  install -m 755 ${ALGO_BIN}/${f} %{buildroot}/usr/bin/${f}
+done
+
+mkdir -p %{buildroot}/etc/pki/rpm-gpg
+install -m 644 ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand %{buildroot}/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand
+
+mkdir -p %{buildroot}/usr/lib/algorand/yum.repos.d
+install -m 644 ${REPO_DIR}/installer/rpm/algorand-devtools/algorand-devtools.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand-devtools.repo
+
+%files
+/usr/bin/carpenter
+/usr/bin/catchupsrv
+/usr/bin/msgpacktool
+/usr/bin/tealcut
+/usr/bin/tealdbg
+/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand
+/usr/lib/algorand/yum.repos.d/algorand-devtools.repo
+

installer/rpm/algorand.spec → installer/rpm/algorand/algorand.spec

@@ -42,16 +42,16 @@ install -m 644 ${REPO_DIR}/installer/algorand.service %{buildroot}/lib/systemd/s
 install -m 644 ${REPO_DIR}/installer/[email protected] %{buildroot}/lib/systemd/system/[email protected]
 
 mkdir -p %{buildroot}/etc/cron.hourly
-install -m 755 ${REPO_DIR}/installer/rpm/0yum-algorand-hourly.cron %{buildroot}/etc/cron.hourly/0yum-algorand-hourly.cron
+install -m 755 ${REPO_DIR}/installer/rpm/algorand/0yum-algorand-hourly.cron %{buildroot}/etc/cron.hourly/0yum-algorand-hourly.cron
 
 mkdir -p %{buildroot}/etc/yum
-install -m 644 ${REPO_DIR}/installer/rpm/yum-cron-algorand.conf %{buildroot}/etc/yum/yum-cron-algorand.conf
+install -m 644 ${REPO_DIR}/installer/rpm/algorand/yum-cron-algorand.conf %{buildroot}/etc/yum/yum-cron-algorand.conf
 
 mkdir -p %{buildroot}/etc/pki/rpm-gpg
 install -m 644 ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand %{buildroot}/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand
 
 mkdir -p %{buildroot}/usr/lib/algorand/yum.repos.d
-install -m 644 ${REPO_DIR}/installer/rpm/algorand.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand.repo
+install -m 644 ${REPO_DIR}/installer/rpm/algorand/algorand.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand.repo
 
 mkdir -p %{buildroot}/var/lib/algorand/genesis
 if [ "%{RELEASE_GENESIS_PROCESS}" != "x" ]; then

package.yaml

@@ -44,8 +44,8 @@ tasks:
 jobs:
   package:
     tasks:
-      - docker.Make.rpm
       - docker.Make.deb
+      - docker.Make.rpm
 
   package-docker:
     tasks:

scripts/build_deb.sh

@@ -52,7 +52,7 @@ mkdir -p "${PKG_ROOT}/usr/bin"
 
 if [ "${VARIATION}" = "" ]; then
     # NOTE: keep in sync with installer/rpm/algorand.spec
-    bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "goal" "kmd" "msgpacktool" "node_exporter" "tealdbg")
+    bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg")
 fi
 
 for bin in "${bin_files[@]}"; do
@@ -118,8 +118,8 @@ mkdir -p "${PKG_ROOT}/DEBIAN"
 debian_files=("control" "preinst" "postinst" "prerm" "postrm" "conffiles")
 for ctl in "${debian_files[@]}"; do
     # Copy first, to preserve permissions, then overwrite to fill in template.
-    cp -a "installer/debian/${ctl}" "${PKG_ROOT}/DEBIAN/${ctl}"
-    < "installer/debian/${ctl}" \
+    cp -a "installer/debian/algorand/${ctl}" "${PKG_ROOT}/DEBIAN/${ctl}"
+    < "installer/debian/algorand/${ctl}" \
       sed -e "s,@ARCH@,${ARCH}," \
           -e "s,@VER@,${VER}," \
           -e "s,@PKG_NAME@,${PKG_NAME}," \

scripts/build_package.sh

@@ -58,7 +58,7 @@ DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "${DEFAULT
 mkdir ${PKG_ROOT}/bin
 
 # If you modify this list, also update this list in ./cmd/updater/update.sh backup_binaries()
-bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealdbg" "update.sh" "updater" "COPYING")
+bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg" "update.sh" "updater" "COPYING")
 for bin in "${bin_files[@]}"; do
     cp ${GOPATHBIN}/${bin} ${PKG_ROOT}/bin
     if [ $? -ne 0 ]; then exit 1; fi

scripts/compute_package_name.sh

@@ -1,12 +1,20 @@
 #!/usr/bin/env bash
 
-# ./compute_package_name <CHANNEL>
+# ./compute_package_name <CHANNEL> <NAME>
+#
+# Examples:
+# ./compute_package_name stable                     -> algorand
+# ./compute_package_name beta                       -> algorand-beta
+# ./compute_package_name nightly algorand-devtools  -> algorand-devtools-nightly
 
-if [ "$1" = "nightly" ]; then
-    echo "algorand-nightly"
-elif [ "$1" = "beta" ]; then
-    echo "algorand-beta"
+CHANNEL=${1:-stable}
+NAME=${2:-algorand}
+
+if [ "$CHANNEL" = beta ]; then
+    echo "$NAME-beta"
+elif [ "$CHANNEL" = nightly ]; then
+    echo "$NAME-nightly"
 else
-    echo "algorand"
+    echo "$NAME"
 fi
 

scripts/release/build/deb/build_deb.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+# shellcheck disable=2038,2045,2064
+
+set -ex
+
+echo
+date "+build_release begin PACKAGE DEB stage %Y%m%d_%H%M%S"
+echo
+
+ARCH="$1"
+OUTDIR="$2"
+CHANNEL="$3"
+PKG_NAME="$4"
+OS_TYPE=$(./scripts/ostype.sh)
+mkdir -p "$OUTDIR/bin"
+VER=${VERSION:-$(./scripts/compute_build_number.sh -f)}
+
+echo "Building debian package $PKG_NAME ($CHANNEL) for '$OS - $ARCH'"
+
+DEFAULTNETWORK=$("./scripts/compute_branch_network.sh")
+DEFAULT_RELEASE_NETWORK=$("./scripts/compute_branch_release_network.sh" "$DEFAULTNETWORK")
+export DEFAULT_RELEASE_NETWORK
+
+PKG_ROOT=$(mktemp -d)
+trap "rm -rf $PKG_ROOT" 0
+
+mkdir -p "$PKG_ROOT/usr/bin"
+
+# NOTE: keep in sync with `./installer/rpm/algorand.spec`.
+if [ "$PKG_NAME" = "algorand-devtools" ]; then
+    BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg")
+    UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades"
+    OUTPUT_DEB="$OUTDIR/algorand_devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb"
+else
+    BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter")
+    UNATTENDED_UPGRADES_FILE="51algorand-upgrades"
+    OUTPUT_DEB="$OUTDIR/algorand_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb"
+fi
+
+for binary in "${BIN_FILES[@]}"; do
+    cp "$GOPATH/bin/$binary" "$PKG_ROOT/usr/bin"
+    chmod 755 "$PKG_ROOT/usr/bin/$binary"
+done
+
+if [ "$PKG_NAME" != "algorand-devtools" ]; then
+    mkdir -p "${PKG_ROOT}/usr/lib/algorand"
+    lib_files=("updater" "find-nodes.sh")
+    for lib in "${lib_files[@]}"; do
+        cp "$GOPATH/bin/$lib" "$PKG_ROOT/usr/lib/algorand"
+        chmod g-w "$PKG_ROOT/usr/lib/algorand/$lib"
+    done
+
+    data_files=("config.json.example" "system.json")
+    mkdir -p "$PKG_ROOT/var/lib/algorand"
+    for data in "${data_files[@]}"; do
+        cp "./installer/$data" "$PKG_ROOT/var/lib/algorand"
+    done
+
+    genesis_dirs=("devnet" "testnet" "mainnet" "betanet")
+    for dir in "${genesis_dirs[@]}"; do
+        mkdir -p "$PKG_ROOT/var/lib/algorand/genesis/$dir"
+        cp "./installer/genesis/$dir/genesis.json" "$PKG_ROOT/var/lib/algorand/genesis/$dir/genesis.json"
+    done
+    cp "./installer/genesis/$DEFAULTNETWORK/genesis.json" "$PKG_ROOT/var/lib/algorand/genesis.json"
+
+    # files should not be group writable but directories should be
+    chmod -R g-w "$PKG_ROOT/var/lib/algorand"
+    find "$PKG_ROOT/var/lib/algorand" -type d | xargs chmod g+w
+
+    SYSTEMD_FILES=("algorand.service" "[email protected]")
+    mkdir -p "$PKG_ROOT/lib/systemd/system"
+    for svc in "${SYSTEMD_FILES[@]}"; do
+        cp "./installer/${svc}" "${PKG_ROOT}/lib/systemd/system"
+        chmod 644 "${PKG_ROOT}/lib/systemd/system/${svc}"
+    done
+fi
+
+mkdir -p "$PKG_ROOT/etc/apt/apt.conf.d"
+cat <<EOF> "$PKG_ROOT/etc/apt/apt.conf.d/$UNATTENDED_UPGRADES_FILE"
+## This file is provided by the Algorand package to configure
+## unattended upgrades for the Algorand node software.
+
+Unattended-Upgrade::Allowed-Origins {
+  "Algorand:$CHANNEL";
+};
+
+Dpkg::Options {
+   "--force-confdef";
+   "--force-confold";
+};
+EOF
+
+mkdir -p "$PKG_ROOT/DEBIAN"
+# Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`.
+CTL_FILES_DIR="./installer/debian/$PKG_NAME"
+for ctl_file in $(ls "$CTL_FILES_DIR"); do
+    # Copy first, to preserve permissions, then overwrite to fill in template.
+    cp -a "$CTL_FILES_DIR/$ctl_file" "$PKG_ROOT/DEBIAN/$ctl_file"
+    < "$CTL_FILES_DIR/$ctl_file" \
+      sed -e "s,@ARCH@,$ARCH," \
+          -e "s,@VER@,$VER," \
+          -e "s,@PKG_NAME@,$PKG_NAME," \
+      > "$PKG_ROOT/DEBIAN/$ctl_file"
+done
+
+# TODO: make `Files:` segments for vendor/... and crypto/libsodium-fork, but reasonably this should be understood to cover all _our_ files and copied in packages continue to be licenced under their own terms
+cat <<EOF> "$PKG_ROOT/DEBIAN/copyright"
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: Algorand
+Upstream-Contact: Algorand developers <[email protected]>
+Source: https://github.com/algorand/go-algorand
+
+Files: *
+Copyright: Algorand developers <[email protected]>
+License: AGPL-3+
+EOF
+
+sed 's/^$/./g' < COPYING | sed 's/^/ /g' >> "$PKG_ROOT/DEBIAN/copyright"
+mkdir -p "$PKG_ROOT/usr/share/doc/$PKG_NAME"
+cp -p "$PKG_ROOT/DEBIAN/copyright" "$PKG_ROOT/usr/share/doc/$PKG_NAME/copyright"
+
+dpkg-deb --build "$PKG_ROOT" "$OUTPUT_DEB"
+
+echo
+date "+build_release end PACKAGE DEB stage %Y%m%d_%H%M%S"
+echo
+

scripts/release/build/deb/package.sh

@@ -16,19 +16,20 @@ export PATH="${GOPATH}":/usr/local/go/bin:"${PATH}"
 pushd "${REPO_ROOT}"
 ./scripts/build_packages.sh "${PLATFORM}"
 
-DEBTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "debtmp")
-trap "rm -rf ${DEBTMP}" 0
+PKG_NAMES=("$ALGORAND_PACKAGE_NAME" "$DEVTOOLS_PACKAGE_NAME")
+for pkg_name in "${PKG_NAMES[@]}"; do
+    DEBTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "debtmp")
+    trap "rm -rf ${DEBTMP}" 0
 
-if ! ./scripts/build_deb.sh "${ARCH}" "${DEBTMP}" "${CHANNEL}"
-then
-    echo "Error building debian package for ${PLATFORM}.  Aborting..."
-    exit 1
-fi
+    if ! ./scripts/release/build/deb/build_deb.sh "${ARCH}" "${DEBTMP}" "${CHANNEL}" "${pkg_name}"
+    then
+        echo "Error building debian package ${pkg_name} for ${PLATFORM}.  Aborting..."
+        exit 1
+    fi
 
-BRANCH=$("./scripts/compute_branch.sh")
-CHANNEL=$("./scripts/compute_branch_channel.sh" "$BRANCH")
+    cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/${pkg_name}_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb"
+done
 
-cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/algorand_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb"
 popd
 
 # build docker release package

scripts/release/build/rpm/package.sh

@@ -6,31 +6,40 @@ echo "Building RPM package"
 
 cd "$(dirname "$0")"/..
 
-REPO_DIR=${HOME}/subhome/go/src/github.com/algorand/go-algorand
+REPO_DIR="$HOME/subhome/go/src/github.com/algorand/go-algorand"
 export REPO_DIR
-DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "${DEFAULTNETWORK}")
+DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "$DEFAULTNETWORK")
 export DEFAULT_RELEASE_NETWORK
 DEFAULTNETWORK=devnet
 export DEFAULT_NETWORK
 ALGO_BIN="$HOME/subhome/go/bin"
 export ALGO_BIN
 
-RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
-trap 'rm -rf ${RPMTMP}' 0
-
 BRANCH=$("$REPO_DIR/scripts/compute_branch.sh")
 CHANNEL=$("$REPO_DIR/scripts/compute_branch_channel.sh" "$BRANCH")
-PKG_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}")
+ALGORAND_PACKAGE_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}")
+DEVTOOLS_PACKAGE_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}" algorand-devtools)
+
+PKG_NAMES=("$ALGORAND_PACKAGE_NAME" "$DEVTOOLS_PACKAGE_NAME")
+for pkg_name in "${PKG_NAMES[@]}"; do
+    RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
+    trap 'rm -rf $RPMTMP' 0
+
+    TEMPDIR=$(mktemp -d)
+    trap 'rm -rf $TEMPDIR' 0
+
+    mkdir "$TEMPDIR/$pkg_name"
+
+    echo "Building rpm package $pkg_name ($CHANNEL)"
 
-TEMPDIR=$(mktemp -d)
-trap 'rm -rf $TEMPDIR' 0
-< "$REPO_DIR/installer/rpm/algorand.spec" \
-    sed -e "s,@PKG_NAME@,${PKG_NAME:-algorand}," \
-        -e "s,@VER@,$FULLVERSION," \
-    > "$TEMPDIR/algorand.spec"
+    < "$REPO_DIR/installer/rpm/$pkg_name/$pkg_name.spec" \
+        sed -e "s,@PKG_NAME@,$pkg_name," \
+            -e "s,@VER@,$FULLVERSION," \
+        > "$TEMPDIR/$pkg_name/$pkg_name.spec"
 
-rpmbuild --define "_rpmdir ${RPMTMP}" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "${TEMPDIR}/algorand.spec"
+    rpmbuild --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "$TEMPDIR/$pkg_name/$pkg_name.spec"
 
-mkdir -p /root/subhome/node_pkg
-cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg
+    mkdir -p /root/subhome/node_pkg
+    cp -p "$RPMTMP"/*/*.rpm /root/subhome/node_pkg
+done