Avoid varying git archive content for ref; rework versioning

Don't include the log --decorate names in the archive via export-subst
because they can change after the fact when new tags or branches are
added for a given hash -- for example, when we created the 0.30.x
branch after tagging 0.30.1.  Archives retrieved before the branch was
created would have a different set of NAMES in _release.py.

Move _release to source_info and add an optional checkout_info module.
source_info contains the (no longer variable) export-subst commit hash
and date values, and checkout_info contains the same data for a git
checkout.  Automatically update checkout_info whenever we're at the
top level of a git source tree, but don't include it in the archives.

Record the base version in version.py explicitly as either a release
version like 0.31, for an actual release (which must be committted
before tagging the release), or a development version like
0.31~ (indicating a version that's always less than 0.31).

Rework bup version to report the lib/version for an actual release, or
that version suffixed with the commit hash when running a non-release,
and add a "+" when uncommitted modifications are detected.  For
example:

      release: 0.31
  non-release: 0.31~4e4b9ba8689c93702743c8ecd49c5a7808a4d717
     modified: 0.31~4e4b9ba8689c93702743c8ecd49c5a7808a4d717+

Drop the --tag argument from bup version since the tags are variable,
and you can always ask git to describe the hash via

  git describe --always HASH

Add dev/refresh, similar to moreutils sponge, to handle file creation
safely, something we may want to deploy more widely (e.g. instead of
the $$/PID based tempfiles in the Makefile).

Thanks to Greg Troxel for reporting the problem.

Signed-off-by: Rob Browning <[email protected]>
Tested-by: Rob Browning <[email protected]>

.gitignore

@@ -12,9 +12,9 @@ memtest
 *.tmp.meta
 /build
 /config/bin/
+/lib/bup/checkout_info.py
 *.swp
 nbproject
 /lib/cmd/bup-*
 /t/sampledata/var/
 /t/tmp/
-/lib/bup/_checkout.py

Makefile

@@ -42,8 +42,8 @@ else
   test_tmp := $(CURDIR)/t/tmp
 endif
 
-initial_setup := $(shell ./configure-version --update $(isok))
-initial_setup := $(call shout,$(initial_setup),Version configuration failed))
+initial_setup := $(shell dev/update-checkout-info lib/bup/checkout_info.py $(isok))
+initial_setup := $(call shout,$(initial_setup),update-checkout-info failed))
 
 config/config.vars: \
   configure config/configure config/configure.inc \
@@ -76,7 +76,7 @@ bup_cmds := \
   $(patsubst cmd/%-cmd.py,cmd/bup-%,$(wildcard cmd/*-cmd.py)) \
   $(patsubst cmd/%-cmd.sh,cmd/bup-%,$(wildcard cmd/*-cmd.sh))
 
-bup_deps := lib/bup/_checkout.py lib/bup/_helpers$(SOEXT) $(bup_cmds)
+bup_deps := lib/bup/_helpers$(SOEXT) $(bup_cmds)
 
 all: $(bup_deps) Documentation/all $(current_sampledata)
 
@@ -135,6 +135,16 @@ install: all
 	$(INSTALL) -pm 0644 \
 		lib/web/*.html \
 		$(dest_libdir)/web/
+	if test -e lib/bup/checkout_info.py; then \
+	    $(INSTALL) -pm 0644 lib/bup/checkout_info.py \
+	        $(dest_libdir)/bup/source_info.py; \
+	else \
+	    ! grep -qF '$$Format' lib/bup/source_info.py; \
+	    $(INSTALL) -pm 0644 lib/bup/source_info.py $(dest_libdir)/bup/; \
+	fi
+
+
+	    $(INSTALL) -pm 0644 lib/bup/checkout_info.py $(dest_libdir)/bup/; \
 
 config/config.h: config/config.vars
 
@@ -148,14 +158,6 @@ lib/bup/_helpers$(SOEXT): \
 	  "import glob; assert(len(glob.glob('lib/bup/build/*/_helpers*$(SOEXT)')) == 1)"
 	cp lib/bup/build/*/_helpers*$(SOEXT) "$@"
 
-lib/bup/_checkout.py:
-	@if grep -F '$Format' lib/bup/_release.py \
-	    && ! test -e lib/bup/_checkout.py; then \
-	  echo "Something has gone wrong; $@ should already exist."; \
-	  echo 'Check "./configure-version --update"'; \
-	  false; \
-	fi
-
 t/tmp:
 	mkdir t/tmp
 
@@ -287,7 +289,7 @@ cmd/bup-%: cmd/%-cmd.sh
 Documentation/all: $(man_roff) $(man_html)
 
 Documentation/substvars: $(bup_deps)
-	echo "s,%BUP_VERSION%,$$(./bup version --tag),g" > $@
+	echo "s,%BUP_VERSION%,$$(./bup version),g" > $@
 	echo "s,%BUP_DATE%,$$(./bup version --date),g" >> $@
 
 Documentation/%.1: Documentation/%.md Documentation/substvars
@@ -326,6 +328,7 @@ clean: Documentation/clean config/bin/python
 	rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll lib/*/*.dll *.exe \
 		.*~ *~ */*~ lib/*/*~ lib/*/*/*~ \
 		*.pyc */*.pyc lib/*/*.pyc lib/*/*/*.pyc \
+		lib/bup/checkout_info.py \
 		randomgen memtest \
 		testfs.img lib/bup/t/testfs.img
 	for x in $$(ls cmd/*-cmd.py cmd/*-cmd.sh | grep -vF python-cmd.sh | cut -b 5-); do \
@@ -339,7 +342,6 @@ clean: Documentation/clean config/bin/python
 	  then umount lib/bup/t/testfs || true; fi
 	rm -rf *.tmp *.tmp.meta t/*.tmp lib/*/*/*.tmp build lib/bup/build lib/bup/t/testfs
 	if test -e t/tmp; then t/force-delete t/tmp; fi
-	./configure-version --clean
 	t/configure-sampledata --clean
         # Remove last so that cleanup tools can depend on it
 	rm -rf config/bin

dev/refresh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -ueo pipefail
+
+# Related: https://joeyh.name/code/moreutils/ sponge
+
+usage()
+{
+    echo 'Usage: refresh [-a] [-v] DEST'
+    echo '       refresh [-a] [-v] -- DEST'
+}
+
+append=''
+verbose=''
+
+while test $# -gt 0; do
+    case $1 in
+        -a) append=true; shift;;
+        -v) verbose=true; shift;;
+        --)
+            if test "$#" -ne 2; then
+                usage 1>&2
+                exit 2
+            fi
+            dest="$2"
+            shift 2
+            ;;
+        -*)
+            usage 1>&2
+            exit 2
+            ;;
+        *)
+            if test "$#" -ne 1; then
+                usage 1>&2
+                exit 2
+            fi
+            dest="$1"
+            shift
+            ;;
+    esac
+done
+
+tmpfile="$(mktemp "$dest.sponge-XXXXXXX")"
+
+clean-up()
+{
+    rm -f "$tmpfile"
+}
+
+trap clean-up EXIT
+
+# Inefficient, but should clone the permissions
+if test -e "$dest"; then
+   cp -Lp "$dest" "$tmpfile"
+fi
+
+if test "$append"; then
+    cat >> "$tmpfile"
+else
+    cat > "$tmpfile"
+fi
+
+if ! cmp -s "$tmpfile" "$dest"; then
+    if test "$verbose"; then
+        echo "Refreshed $dest" 1>&2
+    fi
+    mv "$tmpfile" "$dest"
+fi

dev/update-checkout-info

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+top="$(pwd)"
+
+usage() { echo 'Usage: update-checkout-info DEST'; }
+
+if test "$#" -ne 1; then
+    usage 1>&2; exit 1
+fi
+
+dest="$1"
+
+if ! test -f lib/bup/bupsplit.c; then
+    echo 'error: cannot find bup source tree' 1>&2
+    exit 1
+fi
+
+git_top=$(git rev-parse --show-toplevel) || true
+if test "$git_top" != "$top"; then
+    # Not a checkout, or perhaps we're building from an archive dir
+    # unpacked somewhere in the source tree.
+    rm -f "$dest"
+    exit 0
+fi
+
+local_changes=$(git status --porcelain -uno)
+
+(git log -1 --pretty="commit='%H'%ndate='%ci'"
+ echo -n 'modified='
+ if test "$local_changes"; then echo True; else echo False; fi) \
+    | dev/refresh -v -- "$dest"

lib/bup/.gitattributes

@@ -1 +1 @@
-_release.py  export-subst
+source_info.py  export-subst

lib/bup/compat.py

@@ -16,6 +16,7 @@
     from os import environb as environ
     from os import fsdecode, fsencode
     from shlex import quote
+    ModuleNotFoundError = ModuleNotFoundError
     input = input
     range = range
     str_type = str
@@ -64,6 +65,8 @@ def getcwd():
 
 else:  # Python 2
 
+    ModuleNotFoundError = ImportError
+
     def fsdecode(x):
         return x
 

lib/bup/source_info.py

@@ -0,0 +1,3 @@
+commit='$Format:%H$'
+date='$Format:%ci$'
+modified=False

lib/bup/version.py

@@ -1,9 +1,35 @@
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
+import sys
 
-from bup import _release
+from bup.compat import ModuleNotFoundError
 
-if _release.COMMIT != '$Format:%H$':
-    from bup._release import COMMIT, DATE, NAMES
+from bup import source_info
+try:
+    import bup.checkout_info as checkout_info
+except ModuleNotFoundError:
+    checkout_info = None
+    pass
+
+
+if checkout_info:
+    date = checkout_info.date.encode('ascii')
+    commit = checkout_info.commit.encode('ascii')
+    modified = checkout_info.modified
 else:
-    from bup._checkout import COMMIT, DATE, NAMES
+    date = source_info.date.encode('ascii')
+    commit = source_info.commit.encode('ascii')
+    modified = source_info.modified
+    assert not date.startswith(b'$Format')
+    assert not commit.startswith(b'$Format')
+
+# The ~ in a version is a Debian-style "always less than" marker:
+# https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
+base_version = b'0.31~'
+
+version = base_version
+if version.endswith(b'~'):
+    version += commit
+
+if modified:
+    version += b'+'

lib/cmd/version-cmd.py

@@ -20,56 +20,30 @@
 sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
 
 from bup import compat, options, version
+from bup.io import byte_stream
 
 version_rx = re.compile(r'^[0-9]+\.[0-9]+(\.[0-9]+)?(-[0-9]+-g[0-9abcdef]+)?$')
 
 optspec = """
-bup version [--date|--commit|--tag]
+bup version [--date|--commit]
 --
 date    display the date this version of bup was created
 commit  display the git commit id of this version of bup
-tag     display the tag name of this version.  If no tag is available, display the commit id
 """
 o = options.Options(optspec)
 opt, flags, extra = o.parse(compat.argv[1:])
 
 
-total = (opt.date or 0) + (opt.commit or 0) + (opt.tag or 0)
+total = (opt.date or 0) + (opt.commit or 0)
 if total > 1:
     o.fatal('at most one option expected')
 
-
-def version_date():
-    """Format bup's version date string for output."""
-    return version.DATE.split(' ')[0]
-
-
-def version_commit():
-    """Get the commit hash of bup's current version."""
-    return version.COMMIT
-
-
-def version_tag():
-    """Format bup's version tag (the official version number).
-
-    When generated from a commit other than one pointed to with a tag, the
-    returned string will be "unknown-" followed by the first seven positions of
-    the commit hash.
-    """
-    names = version.NAMES.strip()
-    assert(names[0] == '(')
-    assert(names[-1] == ')')
-    names = names[1:-1]
-    l = [n.strip() for n in names.split(',')]
-    for n in l:
-        if n.startswith('tag: ') and version_rx.match(n[5:]):
-            return n[5:]
-    return 'unknown-%s' % version.COMMIT[:7]
-
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
 
 if opt.date:
-    print(version_date())
+    out.write(version.date.split(' ')[0] + b'\n')
 elif opt.commit:
-    print(version_commit())
+    out.write(version.commit + b'\n')
 else:
-    print(version_tag())
+    out.write(version.version + b'\n')