arangodb · dbortoluzzi · Feb 8, 2016 · Feb 8, 2016 · Feb 9, 2016 · Feb 9, 2016
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,9 @@ Azure_ArangoDB_Cluster.sh
 DigitalOcean_ArangoDB_Cluster.sh
 GoogleComputeEngine_ArangoDB_Cluster.sh
 GoogleComputeEngine_Mesos_Cluster.sh
+vCloud_ArangoDB_Cluster.sh
 digital_ocean/
 azure/
 gce/
 aws/
+*.swp
diff --git a/Docker/ArangoDBClusterWithDocker.sh b/Docker/ArangoDBClusterWithDocker.sh
@@ -48,7 +48,7 @@
 
 startArangoDBClusterWithDocker() {
 
-    DOCKER_IMAGE_NAME=neunhoef/arangodb_cluster:2.6.dev-2.5
+    DOCKER_IMAGE_NAME=m0ppers/arangodb:3.0
 
     # Two docker images are needed: 
     #  microbox/etcd for the agency and
@@ -195,45 +195,28 @@ startArangoDBClusterWithDocker() {
     wait
 
     echo Starting agency...
-    until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -p 4001:4001 -p 7001:7001 --name=agency -e "ETCD_NONO_WAL_SYNC=1" -v $AGENCY_DIR:/data ${DOCKER_IMAGE_NAME} /usr/lib/arangodb/etcd-arango --data-dir /data --listen-client-urls "http://0.0.0.0:4001" --listen-peer-urls "http://0.0.0.0:7001" >/home/$SSH_USER/agency.log"
+    until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -e ARANGO_NO_AUTH=1 -p 4001:8529 --name=agency -v $AGENCY_DIR:/var/lib/arangodb3 ${DOCKER_IMAGE_NAME} --agency.id 0 --agency.size 1 --javascript.v8-contexts 2 --agency.supervision true"
     do
         echo "Error in remote docker run, retrying..."
     done
 
     sleep 1
-    echo Initializing agency...
-    until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --link=agency:agency --rm ${DOCKER_IMAGE_NAME} arangosh --javascript.execute /scripts/init_agency.js > /home/$SSH_USER/init_agency.log"
-    do
-        echo "Error in remote docker run, retrying..."
-    done
-    echo Starting discovery...
-    until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true --link=agency:agency --name discovery ${DOCKER_IMAGE_NAME} arangosh --javascript.execute scripts/discover.js > /home/$SSH_USER/discovery.log"
-    do
-        echo "Error in remote docker run, retrying..."
-    done
 
     start_dbserver () {
         i=$1
         echo Starting DBserver on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER
 
         until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \
-            docker run --detach=true -v $DBSERVER_DATA:/data \
-             -v $DBSERVER_LOGS:/logs --net=host \
+            docker run -p $PORT_DBSERVER:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $DBSERVER_DATA:/var/lib/arangodb3 \
              --name=dbserver$PORT_DBSERVER ${DOCKER_IMAGE_NAME} \
-              arangod --database.directory /data \
-              --frontend-version-check false \
-              --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \
+              arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \
               --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \
-              --server.endpoint tcp://0.0.0.0:$PORT_DBSERVER \
               --cluster.my-local-info dbserver:${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \
-              --log.file /logs/$PORT_DBSERVER.log \
-              --dispatcher.report-interval 15 \
-              --server.foxx-queues false \
-              --server.disable-statistics true \
+              --cluster.my-role PRIMARY \
               --scheduler.threads 3 \
               --server.threads 5 \
-              $DBSERVER_ARGS \
-              >/dev/null
+              --javascript.v8-contexts 6 \
+              $DBSERVER_ARGS
         do
             echo "Error in remote docker run, retrying..."
         done
@@ -244,24 +227,18 @@ startArangoDBClusterWithDocker() {
         echo Starting Coordinator on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR
 
         until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \
-            docker run --detach=true -v $COORDINATOR_DATA:/data \
-                -v $COORDINATOR_LOGS:/logs --net=host \
+            docker run -p $PORT_COORDINATOR:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $COORDINATOR_DATA:/var/lib/arangodb3 \
                 --name=coordinator$PORT_COORDINATOR \
                 ${DOCKER_IMAGE_NAME} \
-              arangod --database.directory /data \
-               --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \
+              arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \
                --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \
-               --server.endpoint tcp://0.0.0.0:$PORT_COORDINATOR \
                --cluster.my-local-info \
                          coordinator:${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \
-               --log.file /logs/$PORT_COORDINATOR.log \
-               --dispatcher.report-interval 15 \
-               --server.foxx-queues false \
-               --server.disable-statistics true \
+               --cluster.my-role COORDINATOR \
                --scheduler.threads 4 \
-               --server.threads 40 \
-               $COORDINATOR_ARGS \
-               >/dev/null
+               --javascript.v8-contexts 11 \
+               --server.threads 10 \
+               $COORDINATOR_ARGS
         do
             echo "Error in remote docker run, retrying..."
         done
@@ -294,84 +271,58 @@ startArangoDBClusterWithDocker() {
             fi
         done
     }
-
-    #for i in `seq 0 $LASTDBSERVER` ; do
-    #    testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER
-    #done
-
-    for i in $COORDINATOR_MACHINES ; do
-        testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR
+
+    DBSERVER_IDS=()
+    for i in `seq 0 $LASTDBSERVER` ; do
+        testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER
+        DBSERVER_IDS[$i]=$(curl http://"${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER"/_admin/server/id)
     done
 
-    echo Bootstrapping DBServers...
-    curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/bootstrapDbServers" \
-         -d '{"isRelaunch":false}' >/dev/null 2>&1
-
-    echo Running DB upgrade on cluster...
-    curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/upgradeClusterDatabase" \
-         -d '{"isRelaunch":false}' >/dev/null 2>&1
-
-    echo Bootstrapping Coordinators...
     for i in $COORDINATOR_MACHINES ; do
-        echo Doing ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR
-        curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR/_admin/cluster/bootstrapCoordinator" \
-             -d '{"isRelaunch":false}' >/dev/null 2>&1 &
+        testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR
     done
-
-    wait
-
+
     if [ ! -z "$REPLICAS" ] ; then
         start_replica () {
             i=$1
             j=`expr $i + 1`
+            ID="Secondary$j"
             if [ $j -gt $LASTDBSERVER ] ; then
                 j=0
             fi
             echo Starting asynchronous replica for
             echo "  ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER on ${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA"
+
+            while true; do
+              curl -f -X PUT --data "{\"primary\": \"${DBSERVER_IDS[$i]}\", \"oldSecondary\": \"none\", \"newSecondary\": \"${ID}\"}" -H "Content-Type: application/json" "${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR"/_admin/cluster/replaceSecondary
+              if [ "$?" == "0" ]; then
+                break
+              fi
+              echo "Failed registering secondary...Retrying..."
+              sleep 1
+            done
 
             until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$j]} $SSH_SUFFIX \
-                docker run --detach=true -v $REPLICA_DATA:/data \
-                 -v $REPLICA_LOGS:/logs --net=host \
+                docker run -p 8529:$PORT_REPLICA --detach=true -e ARANGO_NO_AUTH=1 -v $REPLICA_DATA:/var/lib/arangodb3 \
                  --name=replica$PORT_REPLICA ${DOCKER_IMAGE_NAME} \
-                  arangod --database.directory /data \
-                  --frontend-version-check false \
-                  --server.endpoint tcp://0.0.0.0:$PORT_REPLICA \
-                  --log.file /logs/$PORT_REPLICA.log \
-                  --dispatcher.report-interval 15 \
-                  --server.foxx-queues false \
-                  --server.disable-statistics true \
-                  --scheduler.threads 1 \
-                  --server.threads 2 \
-                  $REPLICA_ARGS \
-                  >/dev/null
+                  arangod \
+                  --cluster.my-id "$ID" \
+                  --cluster.my-role SECONDARY \
+                  --scheduler.threads 3 \
+                  --server.threads 5 \
+                  --javascript.v8-contexts 6 \
+                  $REPLICA_ARGS
             do
                 echo "Error in remote docker run, retrying..."
             done
         }
 
         for i in `seq 0 $LASTDBSERVER` ; do
-            start_replica $i &
+            start_replica $i
         done
 
         echo Waiting 10 seconds till replicas are up and running...
         sleep 10
-
-        for i in `seq 0 $LASTDBSERVER` ; do
-            j=`expr $i + 1`
-            if [ $j -gt $LASTDBSERVER ] ; then
-                j=0
-            fi
-            echo Attaching replica on $j for $i ...
-            curl -s -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-config" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'","database":"_system","includeSystem":false}' --dump -
-            # >/dev/null 2>&1
-            TICK=`curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/sync" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'"}' | sed -e 's/^.*lastLogTick":"\([0-9]*\)"}.*$/\1/'`
-            # >/dev/null 2>&1
-            curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-start?from=$TICK" --dump - && echo
-            # >/dev/null 2>&1
-        done
-    wait
-
     fi
 
     echo ""

diff --git a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh
@@ -12,9 +12,6 @@
 
 trap "kill 0" SIGINT
 
-#CoreOS AWS Image List
-#https://coreos.com/docs/running-coreos/cloud-providers/ec2/
-
 MACHINE_TYPE="t1.medium"
 NUMBER="3"
 OUTPUT="aws"
@@ -126,7 +123,7 @@ done
 
 PREFIX="arangodb-test-$$-"
 echo "OUTPUT DIRECTORY: $OUTPUT"
-echo "ZONE: $ZONE"
+echo "ZONE: $zone"
 echo "PROJECT: $PROJECT"
 
 DEFAULT_KEY_PATH="$HOME/.ssh/${PREFIX}aws-ssh-key"
@@ -141,46 +138,21 @@ else
   ZONE=$zone
 fi
 
-#We have to keep all ami's up to date
-# Current Version: CoreOS 607.0.0
-# URL: https://coreos.com/docs/running-coreos/cloud-providers/ec2/
-
-if [ "$zone" == "eu-central-1" ]; then
-  IMAGE="ami-0e300d13"
-fi
-
-if [ "$zone" == "ap-northeast-1" ]; then
-  IMAGE="ami-af28dcaf"
-fi
-
-if [ "$zone" == "sa-east-1" ]; then
-  IMAGE="ami-2354ec3e"
-fi
-
-if [ "$zone" == "ap-southeast-2" ]; then
-  IMAGE="ami-b9b5c583"
-fi
-
-if [ "$zone" == "ap-southeast-1" ]; then
-  IMAGE="mi-f80b3aaa"
-fi
+# Function to get latest core os ami ids (stable channel)
+# URL: https://coreos.com/dist/aws/aws-stable.json
 
-if [ "$zone" == "us-east-1" ]; then
-  IMAGE="ami-323b195a"
-fi
-
-if [ "$zone" == "us-west-2" ]; then
-  IMAGE="ami-0789a437"
-fi
+echo "Your aws zone is: $ZONE"
+echo "Searching ami id of latest core os version..."
 
-if [ "$zone" == "us-west-1" ]; then
-  IMAGE="ami-8dd533c9"
-fi
-
-if [ "$zone" == "eu-west-1" ]; then
-  IMAGE="ami-55950a22"
-fi
+function jsonval {
+  temp=`echo $json | sed 's/\\\\\//\//g' | sed 's/[{}]//g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | sed 's/\"\:\"/\|/g' | sed 's/[\,]/ /g' | sed 's/\"//g' | grep -w $ZONE`
+  echo ${temp##*|}
+}
 
+json=`curl -s -X GET https://coreos.com/dist/aws/aws-stable.json`
+amiid=`jsonval`
+IMAGE=`echo $amiid | awk {'print $3'}`
+echo "Found image. ID is: $IMAGE"
 
 if test -e "$OUTPUT";  then
   if [ "$REMOVE" == "1" ] ; then
@@ -337,7 +309,7 @@ function setMachineName () {
 function setMachineSecurity () {
   echo "Adding security groups."
   id=`cat "$OUTPUT/temp/IDS$i"`
-  secureid=`aws ec2 describe-security-groups --output json --group-names ${PREFIX}security |python -mjson.tool|grep GroupId| awk {'print $2'}| cut -c 2- | rev | cut -c 3- | rev`
+  secureid=$(aws ec2 describe-security-groups --output json --group-names ${PREFIX}security | grep GroupId | sed -e 's/.*: "\([^"]*\)"/\1/g')
   aws ec2 modify-instance-attribute --instance-id "$id" --groups "$secureid"
 }
 

diff --git a/platformAZURE/Azure_ArangoDB_Cluster.sh b/platformAZURE/Azure_ArangoDB_Cluster.sh
@@ -12,11 +12,11 @@
 
 trap "kill 0" SIGINT
 
-ZONE="West US"
+ZONE="West Europe"
 MACHINE_TYPE="Medium"
 NUMBER="3"
 OUTPUT="azure"
-IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-607.0.0"
+IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-835.12.0"
 SSH_KEY_PATH=""
 DEFAULT_KEY_PATH="$OUTPUT/arangodb_azure_key"
 
@@ -165,7 +165,7 @@ then
     DEFAULT_KEY_PATH="$HOME/.ssh/arangodb_azure_key"
   else
     echo "No SSH-Key-Path given. Creating a new SSH-Key."
-    ssh-keygen -t dsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com"
+    ssh-keygen -t rsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com"
 
 #    openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout "$DEFAULT_KEY_PATH" -out "$DEFAULT_KEY_PATH.pem"
     openssl req -x509 -key "$DEFAULT_KEY_PATH" -nodes -days 365 -newkey rsa:2048 -out "${DEFAULT_KEY_PATH}.pem"

diff --git a/platformDO/DigitalOcean_ArangoDB_Cluster.sh b/platformDO/DigitalOcean_ArangoDB_Cluster.sh
@@ -192,7 +192,7 @@ if test -z "$SSHID";  then
 
   then
     echo "No ArangoDB SSH-Key found. Generating a new one.!"
-    ssh-keygen -t dsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com"
+    ssh-keygen -t rsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com"
 
     if [ $? -eq 0 ]; then
       echo OK

diff --git a/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh b/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh
@@ -375,7 +375,7 @@ export SERVERS_EXTERNAL
 export SERVERS_IDS
 export SSH_USER="core"
 export SSH_CMD="ssh"
-export SSH_SUFFIX="-i $DEFAULT_KEY_PATH -l $SSH_USER"
+export SSH_SUFFIX=""
 export ZONE
 export PROJECT
 export DBSERVER_DATA=/data/dbserver