Docker image buidl scripts

.dockerignore

@@ -0,0 +1,9 @@
+.git
+
+# IDEs
+.idea/
+.vscode/
+*.code-workspace
+
+# build
+/target/

.gitignore

@@ -1,3 +1,7 @@
-/target/
+# IDEs
 .idea/
 .vscode/
+*.code-workspace
+
+# build
+/target/

docker/Dockerfile

@@ -0,0 +1,53 @@
+## SPDX-License-Identifier: Apache-2.0
+# Stage 0
+## Set up and compilation stage
+FROM ubuntu:18.04 AS building
+ENV tempPkgs='\
+    build-essential \
+    ca-certificates \
+    curl \
+    libssl-dev \
+    pkg-config \
+    file \
+    capnproto \
+    '
+RUN set -e; \
+    apt-get update -yq; \
+    apt-get install -yq --no-install-recommends $tempPkgs; \
+    # Install and set up rustup
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2019-09-11 --no-modify-path; 
+WORKDIR /home
+ENV PATH="/root/.cargo/bin:$PATH"
+COPY . native_spark
+RUN set -e; cd native_spark; \
+    echo "PATH: ${PATH}"; \
+    # Build executables
+    cargo build --release --examples; \
+    cp config_files/hosts.conf /root/; \
+    mkdir /home/release; \
+    # Copy all examples binaries
+    find ./target/release/examples -exec file {} \; \
+    | grep "shared object" \
+    | cut -d: -f1 \
+    | grep -v .*- \
+    | xargs -I{} cp "{}" /home/release
+
+# Stage 1
+## Self-contained build with only necessary utils and binaries
+FROM ubuntu:18.04
+WORKDIR /home/release
+COPY --from=building /home/release .
+RUN set -e; \
+    # Install requirements
+    apt-get update -yq; \
+    apt-get install -yq locales ssh capnproto; \
+    # Locales
+    locale-gen en_US.UTF-8; \
+    # Cleanup
+    apt-get purge -y --auto-remove $tempPkgs; \
+    apt-get autoremove -q -y; \
+    apt-get clean -yq; \
+    rm -rf /var/lib/apt/lists/* 
+ENV LANG=en_US.UTF-8  \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8

docker/build_image.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+SCRIPT_PATH=`dirname $(readlink -f $0)`
+
+VERSION=$1
+if [ -z $VERSION ]
+then
+VERSION='latest'
+fi
+PACKAGE="native_spark:${VERSION}"
+
+cd $SCRIPT_PATH && cd ..
+echo "work dir: $(pwd)"
+echo "building $PACKAGE..."
+docker build -t $PACKAGE -f docker/Dockerfile --force-rm .

docker/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3'
+
+networks:
+  native-spark:
+
+services:
+  master:
+    image: native_spark:latest
+    deploy:
+      restart_policy:
+        condition: on-failure
+
+  worker:
+    image: native_spark:latest
+    deploy:
+      restart_policy:
+        condition: on-failure

docker/testing_cluster.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+SCRIPT_PATH=`dirname $(readlink -f $0)`
+cd SCRIPT_PATH
+
+# Deploy a testing cluster with one master and 3 workers
+docker-compose up -d

src/error.rs

@@ -50,4 +50,7 @@ pub enum Error {
 
     #[error("failed to parse slave address {0}")]
     ParseSlaveAddress(String),
+
+    #[error("operation not supported: {0}")]
+    UnsupportedOperation(String),
 }

src/hosts.rs

@@ -36,7 +36,7 @@ mod tests {
     #[test]
     fn test_missing_hosts_file() {
         match Hosts::load_from("/does_not_exist").unwrap_err() {
-            Error::LoadHosts { source: _, path: _ } => {}
+            Error::LoadHosts { .. } => {}
             _ => panic!("Expected Error::LoadHosts"),
         }
     }
@@ -47,7 +47,7 @@ mod tests {
         file.write_all("invalid data".as_ref()).unwrap();
 
         match Hosts::load_from(&path).unwrap_err() {
-            Error::ParseHosts { source: _, path: _ } => {}
+            Error::ParseHosts { .. } => {}
             _ => panic!("Expected Error::ParseHosts"),
         }
     }

src/rdd/mod.rs

@@ -264,15 +264,14 @@ pub trait Rdd<T: Data>: RddBase {
     }
 
     /// Return the first element in this RDD.
-    fn first(&self) -> std::result::Result<T, Box<dyn std::error::Error>>
+    fn first(&self) -> Result<T>
     where
         Self: Sized + 'static,
     {
         if let Some(result) = self.take(1)?.into_iter().next() {
             Ok(result)
         } else {
-            //FIXME: return a proper error when we add return errors
-            panic!("empty collection")
+            Err(Error::UnsupportedOperation("empty collection".to_owned()))
         }
     }
 

tests/test_rdd.rs

@@ -92,11 +92,10 @@ fn test_first() {
     let taken_1 = col1_rdd.first();
     assert!(taken_1.is_ok());
 
-    // TODO: uncomment when it returns a proper error instead of panicking
-    // let col2: Vec<i32> = vec![];
-    // let col2_rdd = sc.parallelize(col2, 4);
-    // let taken_0 = col2_rdd.first();
-    // assert!(taken_0.is_err());
+    let col2: Vec<i32> = vec![];
+    let col2_rdd = sc.parallelize(col2, 4);
+    let taken_0 = col2_rdd.first();
+    assert!(taken_0.is_err());
 }
 
 #[test]