-fully compatible with ew-shopp/ingestion_lib

unimib-datAI · Dec 13, 2019 · 46e854d · 46e854d
1 parent ab68a40
commit 46e854d
Show file tree

Hide file tree

Showing 8 changed files with 136 additions and 59 deletions.
diff --git a/pom.xml b/pom.xml
@@ -50,10 +50,11 @@
 					<skipDocker>false</skipDocker>
 					<resources>
 						<resource>
-							<targetPath>/</targetPath>
-							<directory>${project.build.directory}</directory>
-							<include>${project.build.finalName}.jar</include>
-						</resource>
+                            <targetPath>/</targetPath>
+                            <directory>${project.build.directory}</directory>
+                            <include>${project.build.finalName}.jar</include>
+                            <include>bash_scripts,common_scripts</include>
+                        </resource>
 					</resources>
 				</configuration>
 			</plugin>
@@ -81,20 +82,20 @@
 				</executions>
 				<configuration>
 					<composeFiles>
-						<composeFile>${project.basedir}/src/main/docker/docker-compose.yaml</composeFile>
-					</composeFiles>
-					<detachedMode>true</detachedMode>
-					<removeVolumes>true</removeVolumes>
-					<removeOrphans>true</removeOrphans>
-					<removeImages>true</removeImages>
-					<removeImagesType>local</removeImagesType>
-					<build>false</build>
-					<detail>true</detail>
-					<verbose>false</verbose>
-					<projectName>prova</projectName>
-					<!--<apiVersion>1.22</apiVersion>-->
-					<!--<apiVersion>1.39</apiVersion>-->
-				</configuration>
+                        <composeFile>${project.basedir}/src/main/docker/docker-compose.yaml</composeFile>
+                    </composeFiles>
+                    <detachedMode>true</detachedMode>
+                    <removeVolumes>true</removeVolumes>
+                    <removeOrphans>true</removeOrphans>
+                    <removeImages>true</removeImages>
+                    <removeImagesType>local</removeImagesType>
+                    <build>false</build>
+                    <detail>true</detail>
+                    <verbose>false</verbose>
+                    <projectName>ewsevents</projectName>
+                    <!--<apiVersion>1.22</apiVersion>-->
+                    <!--<apiVersion>1.39</apiVersion>-->
+                </configuration>
 			</plugin>
 		</plugins>
 	</build>

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
@@ -1,16 +1,13 @@
 FROM openjdk:8-jre-alpine
 
-ENV JAVA_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx512m -Xmx256m"
-ENV ARANGO_HOST="arangodb"
-ENV ARANGO_PORT=8529
-ENV ARANGO_USER="root"
-ENV ARANGO_DB_NAME="arango-demo"
-ENV ARANGO_PASSWORD="root"
-ENV REMOTE_API_ENDPOINT="http://conciliator:8080/reconcile/"
+RUN apk add --update bash && rm -rf /var/cache/apk/*
 
-ADD *.jar /app.jar
+RUN mkdir code
+ADD *.jar /code/app.jar
+COPY ./bash_scripts /code
+COPY ./common_scripts /code
 
-RUN sh -c 'touch /app.jar'
+RUN sh -c 'touch /code/app.jar'
 
 EXPOSE 8080
-CMD java -Djava.security.egd=file:/dev/./urandom -jar /app.jar
+ENTRYPOINT ["/code/main.sh"]
diff --git a/src/main/docker/bash_scripts/single_shot.sh b/src/main/docker/bash_scripts/single_shot.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# arg1: run_file_name              
+# arg2: retry_max_count
+# arg3: code directory
+# arg4: input directory
+# arg5: work directory
+# arg6: output directory
+
+code_directory=${3}
+input_directory=${4}
+work_directory=${5}
+output_directory=${6}
+
+echo "code_directory: ${code_directory}"
+echo "input_directory: ${input_directory}"
+echo "work_directory: ${work_directory}"
+echo "output_directory: ${output_directory}"
+
+echo "checking configuration file in input directory"
+nconfFiles="$(find "${input_directory}" -name "*.properties" | wc -l | tr -d '[:space:]' )"
+if [[ "${nconfFiles}" -gt "0" ]]; then
+        # Extract File Name in random pos
+        file_num=`shuf -i1-${nconfFiles} -n1`
+        config_file="$(find "${input_directory}" -name "*.properties" | head "-${file_num}" | tail -1)"
+        echo "// Found ${nconfFiles} Files"
+        echo "// Picking the configuration file num ${file_num}"
+        echo "// File to process ${config_file}"
+fi
+
+echo '***'
+echo '#'
+echo '#  Starting Process: ...'
+echo '#'
+
+# Do processing here
+echo "   Starting java application"
+echo "Running:  java -jar ${code_directory}/app.jar --working_path=${work_directory} --spring.config.location=file:///${config_file} --results_dir=run"
+java -Djava.security.egd=file:/dev/./urandom -jar ${code_directory}/app.jar --working_path=${work_directory} --spring.config.location=file:///${config_file} --results_dir=run
+
+
+# Move the file(s) to output
+${code_directory}/move_to_output.sh ${output_directory} ${work_directory}/run/*
+
+rm -rf ${work_directory}/run
+
+echo '#'
+echo '#  End Process'
+echo '#'
+
diff --git a/src/main/docker/docker-compose.yaml b/src/main/docker/docker-compose.yaml
@@ -1,26 +1,8 @@
 version: '3.7'
 services:
-  arangodb:
-    container_name: arangodb
-    image: arangodb:latest
-    restart: always
-    environment:
-      - ARANGO_ROOT_PASSWORD=openSesame
-    #      - ARANGO_RANDOM_ROOT_PASSWORD=1
-    #      - ARANGO_NO_AUTH=0
-    ports:
-      - 8529:8529
-    volumes:
-      - ../../../arangodb:/var/lib/arangodb3
   app:
     image: miciav/eventsdownloader:0.1
-    environment:
-      - ARANGO_HOST=arangodb
-      - ARANGO_USER=root
-      - ARANGO_PASSWORD=openSesame
-      - ARANGO_EVENTS=events
-      - API=https://cep.cocos.si/nodeproxy/cep/v1/customEvents/event/
-    expose:
-      - 8080
-    depends_on:
-      - arangodb
+    volumes:
+      - ./input:/input
+      - ./output:/output
+    command: ["0", "code", "input", "work", "output"]
diff --git a/src/main/docker/docker-compose_full.yaml b/src/main/docker/docker-compose_full.yaml
@@ -0,0 +1,26 @@
+version: '3.7'
+services:
+  arangodb:
+    container_name: arangodb
+    image: arangodb:latest
+    restart: always
+    environment:
+      - ARANGO_ROOT_PASSWORD=openSesame
+    #      - ARANGO_RANDOM_ROOT_PASSWORD=1
+    #      - ARANGO_NO_AUTH=0
+    ports:
+      - 8529:8529
+    volumes:
+      - ../../../arangodb:/var/lib/arangodb3
+  app:
+    image: miciav/eventsdownloader:0.1
+    environment:
+      - ARANGO_HOST=arangodb
+      - ARANGO_USER=root
+      - ARANGO_PASSWORD=openSesame
+      - ARANGO_EVENTS=events
+      - API=https://cep.cocos.si/nodeproxy/cep/v1/customEvents/event/
+    expose:
+      - 8080
+    depends_on:
+      - arangodb
diff --git a/src/main/java/it/disco/unimib/EWSEventApp.java b/src/main/java/it/disco/unimib/EWSEventApp.java
@@ -55,6 +55,17 @@ public class EWSEventApp implements CommandLineRunner {
     @Value("${spring.data.arangodb.database:''}")
     private String database;
 
+    @Value("${working_path:~}")
+    private String pathName;
+    @Value("${fileName:output.txt}")
+    private String fileName;
+    @Value("${results_dir:results}")
+    private String folderName;
+
+    @Value("${spring.autoconfigure.exclude:null}")
+    private String exclude;
+
+
     public EWSEventApp(StoreEvents storeEvents) {
         this.storeEvents = storeEvents;
     }
@@ -82,13 +93,17 @@ public static boolean isValidFormat(String value, DateTimeFormatter formatter) {
     @Override
     public void run(String... args) throws Exception {
 
-        log.info("ArangoDB host: " + host);
-        log.info("ArangoDB database: " + database);
+        if (!exclude.equalsIgnoreCase("null")) {
+            log.info("ArangoDB host: " + host);
+            log.info("ArangoDB database: " + database);
+        }
         processDateProperties();
         log.info("Starting date: " + startingDate);
         log.info("Ending date: " + endingDate);
         log.info("Number of days: " + NumDays);
-
+        log.info("Path name: " + pathName);
+        log.info("Folder name " + folderName);
+        log.info("File name: " + fileName);
 
         String url = EventEndpointAPI + startingDate;
         if (NumDays >= 0) url += "P" + NumDays;

diff --git a/src/main/java/it/disco/unimib/repository/FileRepository.java b/src/main/java/it/disco/unimib/repository/FileRepository.java
@@ -21,15 +21,20 @@
 @ConditionalOnProperty(name = "outputOnFile", havingValue = "enabled")
 public class FileRepository {
 
+
+    private String pathName;
     private String folderName;
     private String fileName;
     private BufferedWriter bufferedWriter;
+    private Path resultsFullPath;
 
     @Autowired
-    public FileRepository(@Value("${fileName:output.txt}") String fileName,
-                          @Value("${folderName:results}") String folderName) throws IOException {
+    public FileRepository(@Value("${working_path:~}") String pathName,
+                          @Value("${fileName:output.txt}") String fileName,
+                          @Value("${results_dir:results}") String folderName) throws IOException {
         this.fileName = fileName;
         this.folderName = folderName;
+        this.pathName = pathName;
         createOrEmptyFolder();
     }
 
@@ -56,15 +61,16 @@ public void saveAll(Iterable<Event> events) throws IOException {
     }
 
     private void createOrEmptyFolder() throws IOException {
-        File dir = new File(folderName);
+        Path path = Paths.get(pathName, folderName);
+        resultsFullPath = path.toAbsolutePath();
+        File dir = new File(path.toUri());
         if (!dir.exists()) dir.mkdir();
         else FileUtils.cleanDirectory(dir);
     }
 
     private void createFile() throws IOException {
-        Path path = Paths.get(folderName, fileName);
+        Path path = Paths.get(resultsFullPath.toString(), fileName);
         File file = new File(path.toUri());
-
         if (!file.exists()) path = Files.createFile(path);
         bufferedWriter = Files.newBufferedWriter(path);
 

diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -11,5 +11,6 @@ API=https://virtserver.swaggerhub.com/EW-Shopp/EW-Shopp_Event_API/2.2.0/event/
 num_days=5
 starting_date=2018-01-01
 ending_date=2018-02-01
-working_dir=/events
+working_path=/Users/michele
+results_dir=events
 outputOnFile=enabled