Merge pull request #525 from FutureCitiesCatapult/space_syntax_importer

Space syntax importer

src/main/java/uk/org/tombolo/core/utils/FixedValueUtils.java

@@ -51,4 +51,39 @@ public static int save(List<FixedValue> fixedValues){
             return saved;
         });
     }
+
+    /*
+	Save and update requries to check in the database whether the entry exists or not,
+	if exists it updates else adds, but that increase overhead and compute time.
+	Using this method will only keep the old value and discard the new one, in case of 
+	duplicate records.
+	FIXME: Need to find a better way to address it
+	*/
+    public static int saveWithoutUpdate(List<FixedValue> fixedValues){
+        return HibernateUtil.withSession((session) -> {
+            int saved = 0;
+            session.beginTransaction();
+            for (FixedValue fixedValue : fixedValues){
+                try{
+                    session.save(fixedValue);
+                    saved++;
+                }catch(NonUniqueObjectException e){
+                    log.warn("Could not save fixed value for subject {}, attribute {}: {}",
+                            fixedValue.getId().getSubject().getLabel(),
+                            fixedValue.getId().getAttribute().getLabel(),
+                            e.getMessage());
+                }
+                if ( saved % 2000 == 0 ) { 
+                    // FIXME:
+					// Flushing at small intervals increase overhead for the system to clear the session.
+					// The default behaviour of hibernate is to auto flush when it thinks is necessary thus it may be required to 
+					// flush the session manually but this requires testing, and can be cosidered as fixme
+                    session.flush();
+                    session.clear();
+                }
+            }
+            session.getTransaction().commit();
+            return saved;
+        });
+    }
 }

src/main/java/uk/org/tombolo/core/utils/SubjectUtils.java

@@ -1,6 +1,7 @@
 package uk.org.tombolo.core.utils;
 
 import com.vividsolutions.jts.geom.Geometry;
+import org.hibernate.NonUniqueObjectException;
 import org.hibernate.Session;
 import org.hibernate.query.Query;
 import org.slf4j.Logger;
@@ -125,6 +126,38 @@ public static void save(List<Subject> subjects){
 		});
 	}
 
+	/*
+	Save and update requries to check in the database whether the entry exists or not,
+	if exists it updates else adds, but that increase overhead and compute time.
+	Using this method will only keep the old value and discard the new one, in case of 
+	duplicate records.
+	FIXME: Need to find a better way to address it
+	*/
+	public static void saveWithoutUpdate(List<Subject> subjects){
+		HibernateUtil.withSession(session -> {
+			session.beginTransaction();
+			int saved = 0;
+			for (Subject subject : subjects) {
+                try{
+                    session.save(subject);
+                    saved++;
+                }catch(NonUniqueObjectException e){
+                    log.warn("Could not save subject {}, name {},", subject.getLabel(), subject.getName());
+                }
+
+				if ( saved % 2000 == 0 ) {
+					// FIXME:
+					// Flushing at small intervals increase overhead for the system to clear the session.
+					// The default behaviour of hibernate is to auto flush when it thinks is necessary thus it may be required to 
+					// flush the session manually but this requires testing, and can be cosidered as fixme 
+					session.flush();
+					session.clear();
+				}
+			}
+			session.getTransaction().commit();
+		});
+	}
+
 	private static Query queryFromSubjectSpecification(Session session, SubjectRecipe subjectRecipe) {
 		SubjectType subjectType = SubjectTypeUtils.getSubjectTypeByProviderAndLabel(subjectRecipe.getProvider(), subjectRecipe.getSubjectType());
 

src/main/java/uk/org/tombolo/core/utils/TimedValueUtils.java

@@ -122,6 +122,42 @@ public static int save(List<TimedValue> timedValues){
 			return saved;
 		});
 	}
+
+	/*
+	Save and update requries to check in the database whether the entry exists or not,
+	if exists it updates else adds, but that increase overhead and compute time.
+	Using this method will only keep the old value and discard the new one, in case of 
+	duplicate records.
+	FIXME: Need to find a better way to address it
+	*/
+	public static int saveWithoutUpdate(List<TimedValue> timedValues){
+		return HibernateUtil.withSession((session) -> {
+			int saved = 0;
+			session.beginTransaction();
+			for (TimedValue timedValue : timedValues){
+				try{
+					session.save(timedValue);
+					saved++;
+				}catch(NonUniqueObjectException e){
+					log.warn("Could not save timed value for subject {}, attribute {}, time {}: {}",
+							timedValue.getId().getSubject().getLabel(),
+							timedValue.getId().getAttribute().getDescription(),
+							timedValue.getId().getTimestamp().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
+							e.getMessage());
+				}
+				if ( saved % 2000 == 0 ) { 
+					// FIXME:
+					// Flushing at small intervals increase overhead for the system to clear the session.
+					// The default behaviour of hibernate is to auto flush when it thinks is necessary thus it may be required to 
+					// flush the session manually but this requires testing, and can be cosidered as fixme
+					session.flush();
+					session.clear();
+				}
+			}
+			session.getTransaction().commit();
+			return saved;
+		});
+	}
 
 	/**
 	 * FIXME: Supports a very limited number of strings (implemented on-demand)

src/main/java/uk/org/tombolo/importer/ZipUtils.java

@@ -6,28 +6,33 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Enumeration;
+import java.util.UUID;
 import java.util.zip.GZIPInputStream;
 
 public class ZipUtils {
     private static org.slf4j.Logger log = LoggerFactory.getLogger(ZipUtils.class);
 
     public static Path unzipToTemporaryDirectory(File file) throws IOException {
-        ZipFile zipFile = new ZipFile(file);
-        Enumeration<ZipArchiveEntry> zipEntries = zipFile.getEntries();
-        Path tempDirectory = Files.createTempDirectory("temp");
-        while (zipEntries.hasMoreElements()) {
-            ZipArchiveEntry entry = zipEntries.nextElement();
-            FileUtils.copyInputStreamToFile(zipFile.getInputStream(entry), new File(Paths.get(tempDirectory.toString(),"/" + entry.getName()).toString()));
+        File tempDirectory = new File("/tmp/" + UUID.nameUUIDFromBytes(file.getName().getBytes()).toString());
+        if (!tempDirectory.exists()) {
+            tempDirectory.mkdir();
+            ZipFile zipFile = new ZipFile(file);
+            Enumeration<ZipArchiveEntry> zipEntries = zipFile.getEntries();
+            while (zipEntries.hasMoreElements()) {
+                ZipArchiveEntry entry = zipEntries.nextElement();
+                if (!entry.isDirectory()) {
+                    FileUtils.copyInputStreamToFile(zipFile.getInputStream(entry), new File(Paths.get(tempDirectory.toString(), "/" + entry.getName()).toString()));
+                }
+            }
+            zipFile.close();
         }
-
-        zipFile.close();
-        return tempDirectory;
+        return tempDirectory.toPath();
     }
-
     /**
      * Checks if an input stream is gzipped.
      * Gzipped files have a magic number to recognize them.