Merge pull request #495 from devicehive/development

Development

Author: tmatvienko

Jenkinsfile

@@ -1,10 +1,11 @@
 properties([
-  buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '7'))
+  buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '32', daysToKeepStr: '', numToKeepStr: '100'))
 ])
 
 def test_branches = ["development", "master"]
 def publish_branches = ["development", "master"]
 def deploy_branches = ["development"]
+def test_rpc = true
 
 stage('Build jars') {
   node('docker') {
@@ -51,28 +52,30 @@ stage('Build and publish Docker images in CI repository') {
 if (test_branches.contains(env.BRANCH_NAME)) {
   stage('Run integration tests'){
     parallel rpc: {
-      stage('Run integration tests with rpc'){
-        node('tests-runner'){
-          try {
-            clone_devicehive_docker()
-            dir('devicehive-docker/rdbms-image'){
-              writeFile file: '.env', text: """COMPOSE_PROJECT_NAME=ci
-              COMPOSE_FILE=docker-compose.yml:ci-images.yml
-              DH_TAG=${BRANCH_NAME}
-              JWT_SECRET=devicehive
-              DH_FE_SPRING_PROFILES_ACTIVE=rpc-client
-              DH_BE_SPRING_PROFILES_ACTIVE=rpc-server
-              """
-
-              start_devicehive()
+      if (test_rpc) {
+        stage('Run integration tests with rpc'){
+          node('tests-runner'){
+            try {
+              clone_devicehive_docker()
+              dir('devicehive-docker/rdbms-image'){
+                writeFile file: '.env', text: """COMPOSE_PROJECT_NAME=rpc
+                COMPOSE_FILE=docker-compose.yml:ci-images.yml:dh_proxy_custom_certificate.yml:dh_plugin.yml:dh_plugin-ci-image.yml
+                DH_TAG=${BRANCH_NAME}
+                JWT_SECRET=devicehive
+                DH_FE_SPRING_PROFILES_ACTIVE=rpc-client
+                DH_BE_SPRING_PROFILES_ACTIVE=rpc-server
+                """
+
+                start_devicehive()
+              }
+              wait_for_devicehive_is_up()
+              run_devicehive_tests()
+            } finally {
+              archive_container_logs('rpc')
+              zip archive: true, dir: 'devicehive-tests', glob: 'mochawesome-report/**', zipFile: 'mochawesome-report-rpc.zip'
+              shutdown_devicehive()
+              cleanWs()
             }
-            wait_for_devicehive_is_up()
-            run_devicehive_tests()
-          } finally {
-            archive_container_logs('rpc')
-            zip archive: true, dir: 'devicehive-tests', glob: 'mochawesome-report/**', zipFile: 'mochawesome-report.zip'
-            shutdown_devicehive()
-            cleanWs()
           }
         }
       }
@@ -83,8 +86,8 @@ if (test_branches.contains(env.BRANCH_NAME)) {
           try {
             clone_devicehive_docker()
             dir('devicehive-docker/rdbms-image'){
-              writeFile file: '.env', text: """COMPOSE_PROJECT_NAME=ci
-              COMPOSE_FILE=docker-compose.yml:ci-images.yml
+              writeFile file: '.env', text: """COMPOSE_PROJECT_NAME=wsproxy
+              COMPOSE_FILE=docker-compose.yml:ci-images.yml:dh_proxy_custom_certificate.yml:dh_plugin.yml:dh_plugin-ci-image.yml
               DH_TAG=${BRANCH_NAME}
               JWT_SECRET=devicehive
               """
@@ -94,8 +97,8 @@ if (test_branches.contains(env.BRANCH_NAME)) {
             wait_for_devicehive_is_up()
             run_devicehive_tests()
           } finally {
-            archive_container_logs('ws-proxy')
-            zip archive: true, dir: 'devicehive-tests', glob: 'mochawesome-report/**', zipFile: 'mochawesome-report.zip'
+            archive_container_logs('wsproxy')
+            zip archive: true, dir: 'devicehive-tests', glob: 'mochawesome-report/**', zipFile: 'mochawesome-report-wsproxy.zip'
             shutdown_devicehive()
             cleanWs()
           }
@@ -135,6 +138,7 @@ if (deploy_branches.contains(env.BRANCH_NAME)) {
     node('dev-server') {
       dir('/home/centos/devicehive-docker/rdbms-image'){
         sh '''
+          echo "$(date): Deploying build #${BUILD_NUMBER} from ${BRANCH_NAME} branch..." > ./jenkins-cd.timestamp
           sed -i -e "s/DH_TAG=.*/DH_TAG=${BRANCH_NAME}/g" .env
           sudo docker-compose pull
           sudo docker-compose up -d
@@ -149,6 +153,10 @@ def clone_devicehive_docker(){
   dir('devicehive-docker'){
     echo("Clone Docker Compose files")
     git branch: 'development', url: 'https://github.com/devicehive/devicehive-docker.git', depth: 1
+    sh '''
+      mkdir ../ssl || :
+      tar -xakf rdbms-image/tests/localhost_cert.tar.gz -C ../ssl
+    '''
   }
 }
 
@@ -189,7 +197,8 @@ def run_devicehive_tests() {
       jq ".server.ip = \\"127.0.0.1\\"" | \\
       jq ".server.port = \\"80\\"" | \\
       jq ".server.restUrl = \\"http://127.0.0.1/api/rest\\"" | \\
-      jq ".server.authRestUrl = \\"http://127.0.0.1/auth/rest\\"" > config.json
+      jq ".server.authRestUrl = \\"http://127.0.0.1/auth/rest\\"" | \\
+      jq ".server.pluginRestUrl = \\"http://127.0.0.1/plugin/rest\\"" > config.json
     '''
 
     timeout(time:10, unit: 'MINUTES') {
@@ -200,20 +209,21 @@ def run_devicehive_tests() {
 }
 
 def archive_container_logs(flavour) {
-  def logsdir = "${flavour}-container-logs"
+  def logsdir = "container-logs"
   sh """
     mkdir ${logsdir} || :
-    sudo docker logs ci_dh_auth_1 > ${logsdir}/auth.log 2>&1
-    sudo docker logs ci_dh_backend_1 > ${logsdir}/backend.log 2>&1
-    sudo docker logs ci_dh_frontend_1 > ${logsdir}/frontend.log 2>&1
-    sudo docker logs ci_dh_proxy_1 > ${logsdir}/proxy.log 2>&1
-    sudo docker logs ci_hazelcast_1 > ${logsdir}/hazelcast.log 2>&1
-    sudo docker logs ci_kafka_1 > ${logsdir}/kafka.log 2>&1
-    sudo docker logs ci_wsproxy_1 > ${logsdir}/wsproxy.log 2>&1
-    sudo docker logs ci_postgres_1 > ${logsdir}/postgres.log 2>&1
-    sudo docker logs ci_zookeeper_1 > ${logsdir}/zookeeper.log 2>&1
+    sudo docker logs ${flavour}_dh_auth_1 > ${logsdir}/${flavour}_auth.log 2>&1
+    sudo docker logs ${flavour}_dh_backend_1 > ${logsdir}/${flavour}_backend.log 2>&1
+    sudo docker logs ${flavour}_dh_frontend_1 > ${logsdir}/${flavour}_frontend.log 2>&1
+    sudo docker logs ${flavour}_dh_plugin_1 > ${logsdir}/${flavour}_plugin.log 2>&1
+    sudo docker logs ${flavour}_dh_proxy_1 > ${logsdir}/${flavour}_proxy.log 2>&1
+    sudo docker logs ${flavour}_hazelcast_1 > ${logsdir}/${flavour}_hazelcast.log 2>&1
+    sudo docker logs ${flavour}_kafka_1 > ${logsdir}/${flavour}_kafka.log 2>&1
+    sudo docker logs ${flavour}_wsproxy_1 > ${logsdir}/${flavour}_wsproxy.log 2>&1
+    sudo docker logs ${flavour}_postgres_1 > ${logsdir}/${flavour}_postgres.log 2>&1
+    sudo docker logs ${flavour}_zookeeper_1 > ${logsdir}/${flavour}_zookeeper.log 2>&1
   """
-  def logs = "${logsdir}/auth.log, ${logsdir}/backend.log, ${logsdir}/frontend.log, ${logsdir}/proxy.log, ${logsdir}/hazelcast.log, ${logsdir}/kafka.log, ${logsdir}/wsproxy.log, ${logsdir}/postgres.log, ${logsdir}/zookeeper.log"
+  def logs = "${logsdir}/${flavour}_auth.log, ${logsdir}/${flavour}_backend.log, ${logsdir}/${flavour}_frontend.log, ${logsdir}/${flavour}_plugin.log, ${logsdir}/${flavour}_proxy.log, ${logsdir}/${flavour}_hazelcast.log, ${logsdir}/${flavour}_kafka.log, ${logsdir}/${flavour}_wsproxy.log, ${logsdir}/${flavour}_postgres.log, ${logsdir}/${flavour}_zookeeper.log"
   archiveArtifacts artifacts: logs, fingerprint: true
 }
 

README.md

@@ -11,12 +11,11 @@ It provides the communication layer, control software and multi-platform
 libraries to bootstrap development of smart energy, home automation, remote
 sensing, telemetry, remote control and monitoring software and much more.
 
-Connect embedded Linux using Python or C++ libraries and JSON protocol or
-connect AVR, Microchip devices using lightweight C libraries and BINARY protocol.
-Develop client applications using HTML5/JavaScript, iOS and Android libraries.
-For solutions involving gateways, there is also gateway middleware that allows
-to interface with devices connected to it. Leave communications to DeviceHive
-and focus on actual product and innovation.
+Connect embedded Linux using Python, Node.js or Java libraries and JSON format.
+Write and read your data via REST, Websockets or MQTT, explore visualization on [Grafana](https://grafana.com/plugins/devicehive-devicehive-datasource/installation) charts.
+
+Develop client applications using HTML5/JavaScript and Android libraries.
+Leave communications to DeviceHive and focus on actual product and innovation.
 
 DeviceHive license
 ------------------
@@ -32,35 +31,42 @@ highly encouraged to do so!
 
 Docker Container
 =========================================
-We have published a DeviceHive docker container so you can utilize docker's virtualization features with DeviceHive. 
-Check out [DeviceHive on Docker Hub](https://hub.docker.com/r/devicehive/devicehive/) with the instructions on 
-how to use it. You can check dockerfile implementation as well as the script for setting up a new instance running 
-under nginx on [DeviceHive Docker](https://github.com/devicehive/devicehive-docker) 
+DeviceHive could be deployed manually, via Docker Compose or to Kubernetes cluster.
+Our suggestion is to start from [Docker Compose](https://docs.docker.com/compose/) - the easiest way to start your 
+mastering DeviceHive capabilities. Instructions could be found [here](https://github.com/devicehive/devicehive-docker/tree/master/rdbms-image).
+In case you're more familiar with [Kubernetes](https://kubernetes.io/), please follow this 
+[link](https://github.com/devicehive/devicehive-docker/tree/master/k8s) for detailed instructions. 
 
 DeviceHive Java installation instructions
 =========================================
 
+Though docker-compose installation is the most developer-friendly way of running DeviceHive locally, sometimes it's required
+to build and start project manually. Below you can find detailed instructions on that.
+
 Prerequisites
 -------------
 In order to use DeviceHive framework you must have the following components installed and configured:
 * [PostgreSQL 9.1](http://www.postgresql.org/download/) or above.
 * [Apache Kafka 0.10.0.0](http://kafka.apache.org/downloads.html) or above.
-* [Oracle JDK 8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or [OpenJDK 8](http://openjdk.java.net/)
-* [Maven](http://maven.apache.org/download.cgi)
-* [DeviceHiveJava source files](https://github.com/devicehive/devicehive-java-server). This is the main part of the [DeviceHive] framework
+* [DeviceHive Websocket Proxy](https://github.com/devicehive/devicehive-ws-proxy) running (relies on Kafka, 
+so should be started only when Kafka is up and running).
+* [Hazelcast IMDG](https://hazelcast.com/use-cases/imdg/).
+* [Oracle JDK 8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or [OpenJDK 8](http://openjdk.java.net/).
+* [Maven](http://maven.apache.org/download.cgi).
+* [DeviceHiveJava source files](https://github.com/devicehive/devicehive-java-server). This is the main part of the [DeviceHive] framework.
 
 
 Build packages
 --------------
-* Download source code from [GitHub](https://github.com/devicehive/devicehive-java-server) using "Download ZIP" button.
+Download source code from [GitHub](https://github.com/devicehive/devicehive-java-server) using "Download ZIP" button.
 It should always point to recent stable or beta release, but you always can get any other tag or branch.
 It also can be done using one of [Git version control client](http://git-scm.com/downloads/guis) or git command line tool.
 If you prefer git, clone project using command
 
 `git clone https://github.com/devicehive/devicehive-java-server.git`
 
 After that you can switch to the tag or branch you need. The list of all available releases can be found at
-https://github.com/devicehive/devicehive-java-server/releases
+https://github.com/devicehive/devicehive-java-server/releases.
 Execute following command from ${devicehive-java-server-directory}.
 
 `mvn clean package`
@@ -75,7 +81,7 @@ You need to update zookeeper.connect (zookeeper's contact point) and bootstrap.s
 
 Running Hazelcast
 -----------------------
-To start download Hazelcast IMDG 3.8.1 from official site (`https://hazelcast.org/download/`), extract to local drive and create in Hazelcast bin folder file hzstart.sh with folling contents:
+To start, download Hazelcast IMDG 3.8.1 from official site (`https://hazelcast.org/download/`), extract to local drive and create in Hazelcast bin folder file hzstart.sh with following contents:
 
 ```bash
 export JAVA_OPTS="$JAVA_OPTS -cp /path/to/jar/from/devicehive-hazelcast/devicehive-common-<version>-shade.jar:/path/to/HAZELCAST_HOME/lib/hazelcast-all-3.8.1.jar"
@@ -122,11 +128,7 @@ in hazelcast.xml localted in bin folder of hazelcast. Also replace all the map a
 ```
 
 
-Run hzstart.sh. At this ensure that correct value of property hazelcast.cluster.members is installed in
-
-`/path/to/devicehive-java-server/devicehive-backend/src/main/resources/application.properties`
-
-You can also pass this property in JAVA_OPTS when running devicehive-backend.
+Run hzstart.sh.
 
 Starting database
 ---------------------
@@ -137,13 +139,38 @@ the username is `postgres` and the password is `12345`. You can change this in t
 database.
 * Database schema will be initialized on application startup.
 
+Checking properties
+---------------------
+
+Each microservice has its own `src/main/resources/application.properties` file which contains all application-level 
+configurations (db credentials, hazelcast address, kafka props etc.). Please check them before building application in 
+order to avoid problems at runtime.
+
+You can also override these values by passing them to JVM while running `java -Dapplication.property.name=application.property.name -jar`.
+For example: 
+```
+java -Dhazelcast.cluster.members=0.0.0.1:5701 -jar ${devicehive-jar}.jar
+java -Dbootstrap.servers=0.0.0.1:9092 -jar ${devicehive-jar}.jar
+java -Dproxy.connect=0.0.0.1:3000 -jar ${devicehive-jar}.jar
+```
+
+DB connection properties are managed inside `devicehive-rdbms-dao/src/main/resources/application-persistence.properties`.
+To override them do the same:
+```
+java -Dspring.datasource.url=jdbc:postgresql://0.0.0.1:5432/devicehive -jar ${devicehive-jar}.jar
+java -Dspring.datasource.username=test -Dspring.datasource.password=test -jar ${devicehive-jar}.jar
+```
+
 Running application
 ---------------------
-* To start application, you have to start backend, frontend and auth microservices. To do this, first run following command:
+DeviceHive ecosystem contains of 3 mandatory and 1 optional services, namely Backend, Frontend, Auth and Plugin 
+management (optional) micro services.
+
+* To start application, first run following command:
 
 `java -jar ${devicehive-java-server-directory}/devicehive-backend/target/devicehive-backend-<version>-boot.jar`
 
-Wait for the application to start, then run: 
+This will start Backend. Wait for the application to start, then run: 
 
 `java -jar ${devicehive-java-server-directory}/devicehive-frontend/target/devicehive-frontend-<version>-boot.jar`
 
@@ -152,13 +179,26 @@ and
 `java -jar ${devicehive-java-server-directory}/devicehive-auth/target/devicehive-auth-<version>-boot.jar`
 
 This will start embedded undertow application server on default port 8080 and deploy DeviceHive application.
-You can visit http://localhost:8080/dh/swagger from your web browser to start learning the frontend's APIs.
-Also you can visit http://localhost:8090/dh/swagger from your web browser to start learning the auth's APIs.
+You can visit `http://localhost:8080/dh/swagger` from your web browser to start learning the frontend's APIs.
+Also you can visit `http://localhost:8090/dh/swagger` from your web browser to start learning the auth's APIs.
 
 For devicehive-frontend and devicehive-backend logging level can be changed by adding the following properties to the command above:
 
 `-Droot.log.level=value1 -Dcom.devicehive.log.level=value2`
 
 The values can be: TRACE, DEBUG, INFO, WARN, ERROR. If the properties are absent the default values will be used.
 For devicehive-frontend and devicehive-auth default values for value1 and value2 are WARN and INFO correspondingly.
-For devicehive-backend the default value for both is INFO.
+For devicehive-backend the default value for both is INFO.
+
+Plugin management service
+---------------------
+
+There's one optional service inside DeviceHive ecosystem - Plugin Management service. It allows to register and to update 
+DeviceHive plugins (that allow customers to implement their own business logic without diving into DeviceHive source code)
+via RESTful API.
+
+To start it simply run following command:
+
+`java -jar ${devicehive-java-server-directory}/devicehive-plugin/target/devicehive-plugin-<version>-boot.jar`
+
+Service will be started on 8110 port by default, so you can visit its swagger at `http://localhost:8110/dh/swagger`

devicehive-auth/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>devicehive-server</artifactId>
         <groupId>com.devicehive</groupId>
-        <version>3.4.3</version>
+        <version>3.4.4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>
@@ -20,7 +20,7 @@
     <dependencies>
         <dependency>
             <groupId>com.devicehive</groupId>
-            <artifactId>devicehive-common-auth</artifactId>
+            <artifactId>devicehive-common-service</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
         <dependency>
@@ -143,5 +143,31 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>shim-kafka</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.devicehive</groupId>
+                    <artifactId>devicehive-shim-kafka-impl</artifactId>
+                    <version>${project.parent.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
+        <profile>
+            <id>proxy-ws-kafka</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.devicehive</groupId>
+                    <artifactId>devicehive-proxy-ws-kafka-impl</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
     </profiles>
 </project>