Add source ids observability query

This observability query allows you to see how
many sources there are in a cluster and what their
ids are.

examples/pony/talphabet/.gitignore

@@ -0,0 +1,7 @@
+alphabet
+sender
+sent.txt
+alphabet.out
+alphabet.d
+_expected.json
+_test.txt

examples/pony/talphabet/Makefile

@@ -0,0 +1,58 @@
+# include root makefile
+ifndef ROOT_MAKEFILE_MK
+include ../../../Makefile
+endif
+
+# prevent rules from being evaluated/included multiple times
+ifndef $(abspath $(lastword $(MAKEFILE_LIST)))_MK
+$(abspath $(lastword $(MAKEFILE_LIST)))_MK := 1
+
+
+# The following are control variables that determine what logic from `rules.mk` is enabled
+
+# `true`/`false` to enable/disable the actual unit test command so it can be overridden (the targets are still created)
+# applies to both the pony and elixir test targets
+$(abspath $(lastword $(MAKEFILE_LIST)))_UNIT_TEST_COMMAND := false
+
+# `true`/`false` to enable/disable generate pony related targets (build/test/clean) for pony sources in this directory
+# otherwise targets only get created if there are pony sources (*.pony) in this directory.
+$(abspath $(lastword $(MAKEFILE_LIST)))_PONY_TARGET := true
+
+# `true`/`false` to enable/disable generate final file build target using ponyc command for the pony build target so
+# it can be overridden manually
+$(abspath $(lastword $(MAKEFILE_LIST)))_PONYC_TARGET := true
+
+# `true`/`false` to enable/disable generate exs related targets (build/test/clean) for elixir sources in this directory
+# otherwise targets only get created if there are elixir sources (*.exs) in this directory.
+$(abspath $(lastword $(MAKEFILE_LIST)))_EXS_TARGET := true
+
+# `true`/`false` to enable/disable generate docker related targets (build/push) for a Dockerfile in this directory
+# otherwise targets only get created if there is a Dockerfile in this directory
+$(abspath $(lastword $(MAKEFILE_LIST)))_DOCKER_TARGET := true
+
+# `true`/`false` to enable/disable recursing into Makefiles of subdirectories if they exist
+# (and by recursion every makefile in the tree that is referenced)
+$(abspath $(lastword $(MAKEFILE_LIST)))_RECURSE_SUBMAKEFILES := true
+
+
+ALPHABET_PONY_PATH := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+# standard rules generation makefile
+include $(rules_mk_path)
+
+integration-tests-examples-pony-alphabet: alphabet_pony_test
+
+alphabet_pony_test:
+	cd $(ALPHABET_PONY_PATH) && \
+	python _test/gen.py && \
+	integration_test --framed-file-sender _test.txt \
+		--validation-cmd 'python _test/validate.py --expected _expected.json --output' \
+		--log-level error \
+		--batch-size 10 \
+		--output 'received.txt' \
+		--command './alphabet' \
+		--workers 4 \
+		--sink-expect 1000
+
+# end of prevent rules from being evaluated/included multiple times
+endif

examples/pony/talphabet/README.md

@@ -0,0 +1,111 @@
+# Alphabet
+
+This is an example application that will count the number of "votes" sent for
+each letter of the alphabet and send out the previous count for that letter at the end of each update.
+
+## Prerequisites
+
+- ponyc
+- pony-stable
+- Wallaroo
+
+See [Wallaroo Environment Setup Instructions](https://github.com/WallarooLabs/wallaroo/book/getting-started/setup.md).
+
+## Building
+
+Build Alphabet with
+
+```bash
+make
+```
+
+## Generating Data
+
+A data generator is bundled with the application. Use it to generate a file with a fixed number of psuedo-random votes:
+
+```
+cd data_gen
+./data_gen --message-count 1000
+```
+
+This will create a `votes.msg` file in your current working directory.
+
+## Running Alphabet
+
+You will need five separate shells to run this application. Open each shell and go to the `examples/pony/alphabet` directory.
+
+### Shell 1: Metrics
+
+Start up the Metrics UI if you don't already have it running:
+
+```bash
+docker start mui
+```
+
+You can verify it started up correctly by visiting [http://localhost:4000](http://localhost:4000).
+
+If you need to restart the UI, run:
+
+```bash
+docker restart mui
+```
+
+When it's time to stop the UI, run:
+
+```bash
+docker stop mui
+```
+
+If you need to start the UI after stopping it, run:
+
+```bash
+docker start mui
+```
+
+### Shell 2: Data Receiver
+
+Start a listener
+
+```bash
+../../../../giles/receiver/receiver --listen 127.0.0.1:7002 --no-write \
+  --ponythreads=1 --ponynoblock
+```
+
+### Shell 3: Alphabet
+Start the application
+
+```bash
+./alphabet --in 127.0.0.1:7010 --out 127.0.0.1:7002 --metrics 127.0.0.1:5001 \
+  --control 127.0.0.1:12500 --data 127.0.0.1:12501 --external 127.0.0.1:5050 \
+  --cluster-initializer --ponynoblock --ponythreads=1
+```
+
+### Shell 4: Sender
+
+Start a sender
+
+```bash
+../../../../giles/sender/sender --host 127.0.0.1:7010 \
+  --file data_gen/votes.msg \ --batch-size 5 --interval 100_000_000 \
+  --messages 150000000 --binary --variable-size --repeat --ponythreads=1 \
+  --ponynoblock --no-write
+```
+
+## Shutdown
+
+### Shell 5: Shutdown
+
+You can shut down the cluster with this command at any time:
+
+```bash
+cd ~/wallaroo-tutorial/wallaroo/utils/cluster_shutdown
+./cluster_shutdown 127.0.0.1:5050
+```
+
+You can shut down Giles Sender and Giles Receiver by pressing Ctrl-c from their respective shells.
+
+You can shut down the Metrics UI with the following command:
+
+```bash
+docker stop mui
+```

examples/pony/talphabet/_test/gen.py

@@ -0,0 +1,35 @@
+# Copyright 2017 The Wallaroo Authors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+#  implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+
+from json import dumps
+from random import choice, randrange
+from string import lowercase
+from struct import pack
+
+# Construct input data list, and total votes as a dict
+expected = {}
+data = []
+for x in xrange(1000):
+    c = choice(lowercase)
+    v = randrange(1,10000)
+    data.append(pack('>IsI',5, c, v))
+    expected[c] = expected.get(c, 0) + v
+
+with open('_test.txt', 'wb') as fin:
+    for v in data:
+        fin.write(v)
+
+with open('_expected.json', 'wb') as fout:
+    fout.write(dumps(expected))

examples/pony/talphabet/_test/validate.py

@@ -0,0 +1,52 @@
+# Copyright 2017 The Wallaroo Authors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+#  implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+
+import argparse
+from json import loads
+from struct import calcsize, unpack
+
+fmt = '>LsQ'
+def decoder(bs):
+    return unpack(fmt, bs)[1:3]
+
+def pre_processor(decoded):
+    totals = {}
+    for c, v in decoded:
+        totals[c] = v
+    return totals
+
+parser = argparse.ArgumentParser('Alphabet validator')
+parser.add_argument('--output', type=argparse.FileType('rb'),
+                    help="The output file of the application.")
+parser.add_argument('--expected', type=argparse.FileType('r'),
+                    help=("A file containing the expected final vote tally as "
+                          "JSON serialised dict."))
+args = parser.parse_args()
+
+chunk_size = calcsize(fmt)
+decoded = []
+while True:
+    chunk = args.output.read(chunk_size)
+    if not chunk:
+        break
+    decoded.append(decoder(chunk))
+processed = pre_processor(decoded)
+
+expected = loads(args.expected.read())
+for k in expected.keys():
+    s_key = str(k)
+    expected[s_key] = expected.pop(k)
+
+assert(expected == processed)