sdfs Glacier updates for 3.1.

3.1 Final release

install-packages/deb/usr/share/sdfs/mount.sdfs

@@ -1,6 +1,7 @@
 #!/bin/bash
 modprobe fuse > /dev/null
-MEMORY="1000"
+BASEMEMORY="1000"
+CHUNKSZ="80"
 CFG=""
 MPTG=4
 MU="M"
@@ -30,12 +31,14 @@ fi
 if [ ! -n "$MEM" ]; then
 if [ -n "$CFG" ] && [ -f "$CFG" ]; then
         ac=$(echo 'cat //subsystem-config/local-chunkstore/@allocation-size' | xmllint --shell "$CFG" | grep -v ">" | cut -f 2 -d "=" | tr -d - | tr -d \");
-        MEMORY=$(((ac/10737418240*MPTG)+1000))
+        bac=$(echo 'cat //subsystem-config/local-chunkstore/extended-config/@io-threads' | xmllint --shell "$CFG" | grep -v ">" | cut -f 2 -d "=" | tr -d - | tr -d \");
+        MEMORY=$(((ac/10737418240*MPTG)+BASEMEMORY+(bac*CHUNKSZ)))
 fi
 else
         MEMORY=$MEM
 fi
 
+
 LD_PRELOAD="/usr/share/sdfs/bin/libfuse.so.2" $EXEC -server -outfile '&1' -errfile '&2' -Djava.library.path=/usr/share/sdfs/bin/ -home /usr/share/sdfs/bin/jre -Dorg.apache.commons.logging.Log=fuse.logging.FuseLog\
  -Dfuse.logging.level=INFO -Xmx$MEMORY$MU -Xms$MEMORY$MU \
 -XX:+DisableExplicitGC -pidfile /var/run/$PF -XX:+UseG1GC -Djava.awt.headless=true \

src/log4j.properties

@@ -0,0 +1 @@
+log4j.logger.com.amazonaws=WARN

src/org/opendedup/collections/ProgressiveFileBasedCSMap.java

@@ -138,9 +138,10 @@ private ProgressiveFileByteArrayLongMap createWriteMap() throws IOException {
 				File f = new File(fileName + "-" + guid + ".keys");
 				if (!f.exists()) {
 					activeWMap = new ProgressiveFileByteArrayLongMap(fileName + "-" + guid, this.hashTblSz);
+					activeWMap.activate();
 					activeWMap.setUp();
 					this.maps.add(activeWMap);
-					activeWMap.activate();
+
 					written = true;
 				}
 			}
@@ -297,7 +298,13 @@ public synchronized long claimRecords(SDFSEvent evt, LargeBloomFilter bf) throws
 							p = m.nextKeyValue();
 						}
 						int mapsz = maps.size();
+						l = this.gcLock.writeLock();
+						l.lock();
+						try{
 						maps.remove(m);
+						}finally {
+						l.unlock();
+						}
 						mapsz = mapsz - maps.size();
 						SDFSLogger.getLog()
 								.info("removing map " + m.toString() + " sz=" + maps.size() + " rm=" + mapsz);
@@ -320,7 +327,13 @@ public synchronized long claimRecords(SDFSEvent evt, LargeBloomFilter bf) throws
 							p = m.nextKeyValue();
 						}
 						int mapsz = maps.size();
+						l = this.gcLock.writeLock();
+						l.lock();
+						try{
 						maps.remove(m);
+						}finally {
+						l.unlock();
+						}
 						mapsz = mapsz - maps.size();
 						SDFSLogger.getLog()
 								.info("removing map " + m.toString() + " sz=" + maps.size() + " rm=" + mapsz);
@@ -495,10 +508,12 @@ public long setUp() throws Exception {
 				if (!f.exists()) {
 					ProgressiveFileByteArrayLongMap activeWMap = new ProgressiveFileByteArrayLongMap(
 							fileName + "-" + guid, this.hashTblSz);
+					activeWMap.activate();
 					activeWMap.setUp();
+
 					this.maps.add(activeWMap);
 					written = true;
-					activeWMap.activate();
+
 					this.activeWriteMap = activeWMap;
 				}
 			}
@@ -619,7 +634,7 @@ public boolean put(ChunkData cm, boolean persist) throws IOException, HashtableF
 		} finally {
 			try {
 				if (bm != null) {
-					bm.inActive();
+					bm.activate();
 				}
 			} catch (Exception e) {
 

src/org/opendedup/logging/SDFSLogger.java

@@ -6,6 +6,7 @@
 
 
 
+
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.Level;
@@ -17,6 +18,7 @@
 public class SDFSLogger {
 
 	private static Logger log = Logger.getLogger("sdfs");
+	private static Logger awslog = Logger.getLogger("com.amazonaws");
 	private static Logger fslog = Logger.getLogger("fs");
 	private static Logger basicLog = Logger.getLogger("bsdfs");
 	private static boolean debug = false;
@@ -25,7 +27,7 @@ public class SDFSLogger {
 		ConsoleAppender bapp = new ConsoleAppender(new PatternLayout("%m%n"));
 		basicLog.addAppender(bapp);
 
-		basicLog.setLevel(Level.INFO);
+		basicLog.setLevel(Level.WARN);
 		RollingFileAppender app = null;
 		try {
 
@@ -36,6 +38,9 @@ public class SDFSLogger {
 		} catch (IOException e) {
 			log.debug("unable to change appender", e);
 		}
+		awslog.setLevel(Level.WARN);
+		awslog.removeAllAppenders();
+		awslog.addAppender(app);
 		log.setLevel(Level.INFO);
 		fsdebug = true;
 		fslog.setLevel(Level.DEBUG);

src/org/opendedup/sdfs/Main.java

@@ -24,8 +24,8 @@ public class Main {
 	public static boolean checkArchiveOnOpen= false;
 	public static boolean checkArchiveOnRead = false;
 
-	public static int writeTimeoutSeconds = 300;
-	public static int readTimeoutSeconds = 300;
+	public static int writeTimeoutSeconds = -1; //1 hour timeout
+	public static int readTimeoutSeconds = -1; //1 hour timeout
 	// public static VolumeConfigWriterThread wth = null;
 	public static boolean runConsistancyCheck = false;
 

src/org/opendedup/sdfs/filestore/HashBlobArchive.java

@@ -447,7 +447,7 @@ public void onRemoval(RemovalNotification<Long, HashBlobArchive> removal) {
 						removal.getValue().removeCache();
 					}
 				}).build(new CacheLoader<Long, HashBlobArchive>() {
-					public HashBlobArchive load(Long hashid) throws IOException {
+					public HashBlobArchive load(Long hashid) throws Exception {
 						try {
 							HashBlobArchive har = null;
 							File f = getPath(hashid);
@@ -458,9 +458,12 @@ public HashBlobArchive load(Long hashid) throws IOException {
 								har = new HashBlobArchive(f, hashid);
 							har.cached = true;
 							return har;
-						} catch (Exception e) {
+						}catch (DataArchivedException e) {
+							throw e;
+						}
+						catch (Exception e) {
 							SDFSLogger.getLog().error("unable to fetch block [" + hashid + "]", e);
-							throw new IOException("unable to read " + hashid);
+							throw e;
 						}
 					}
 				});

src/org/opendedup/sdfs/filestore/cloud/BatchAwsS3ChunkStore.java

@@ -1169,7 +1169,7 @@ else if (lz4compress) {
 
 		String hast = new String(data);
 		SDFSLogger.getLog()
-				.info("reading hashes " + (String) mp.get("objects")
+				.debug("reading hashes " + (String) mp.get("objects")
 						+ " from " + hid + " encn " + sobj.getKey().substring(5));
 		StringTokenizer ht = new StringTokenizer(hast, ",");
 		StringResult st = new StringResult();

src/org/opendedup/sdfs/io/WritableCacheBuffer.java

@@ -665,6 +665,7 @@ public void commandArchiveException(DataArchivedException e) {
 			l.setMaxSize(fs.size());
 			Finger.FingerPersister fp = new Finger.FingerPersister();
 			fp.l = l;
+			fp.fingers = fs;
 			fp.dedup = df.mf.isDedup();
 			lexecutor.execute(fp);
 			int wl = 0;