-
Notifications
You must be signed in to change notification settings - Fork 79
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding compaction for variable length deduplication
- Loading branch information
Showing
11 changed files
with
3,097 additions
and
3,088 deletions.
There are no files selected for viewing
1,024 changes: 512 additions & 512 deletions
1,024
src/org/opendedup/sdfs/filestore/AWSS3ChunkStore.java
Large diffs are not rendered by default.
Oops, something went wrong.
326 changes: 163 additions & 163 deletions
326
src/org/opendedup/sdfs/filestore/AbstractChunkStore.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,164 +1,164 @@ | ||
package org.opendedup.sdfs.filestore; | ||
|
||
import java.io.IOException; | ||
|
||
import org.opendedup.collections.DataArchivedException; | ||
import org.w3c.dom.Element; | ||
|
||
/** | ||
* | ||
* @author Sam Silverberg The AbstractChunkStore is used as in interface to | ||
* store deduped chunks of data from clients to disk or some other | ||
* storage device or service for later retrieval. Within the chunk store | ||
* service itself data passes as follows | ||
* | ||
* client -> TCP -> Client Thread ->HashChunkService -> TCHashStore -> | ||
* AbstactChunkStore | ||
* | ||
* @see FileChunkStore | ||
* @see S3ChunkStore | ||
* @see NullChunkStore | ||
* @see TCChunkStore | ||
* | ||
*/ | ||
|
||
public interface AbstractChunkStore { | ||
|
||
public String restoreBlock(long id, byte[] hash) throws IOException; | ||
|
||
public boolean blockRestored(String id); | ||
|
||
public abstract long getFreeBlocks(); | ||
|
||
public abstract int getReadSpeed(); | ||
|
||
public abstract int getWriteSpeed(); | ||
|
||
public abstract long getCacheSize(); | ||
|
||
public abstract long getMaxCacheSize(); | ||
|
||
public abstract void setReadSpeed(int kbps); | ||
|
||
public abstract void setWriteSpeed(int kbps); | ||
|
||
public abstract void setCacheSize(long sz) throws IOException; | ||
|
||
/** | ||
* Closes the chunk store, if this is required | ||
*/ | ||
public abstract void close(); | ||
|
||
/** | ||
* Closes the chunk store, if this is required | ||
*/ | ||
public abstract void init(Element config) throws IOException; | ||
|
||
/** | ||
* Each chunk store must have a unique name. | ||
* | ||
* @return returns the unique name of the chunk store | ||
*/ | ||
public abstract String getName(); | ||
|
||
/** | ||
* Sets the name of the chunk store | ||
* | ||
* @param name | ||
* the name of the chunk store. | ||
*/ | ||
public abstract void setName(String name); | ||
|
||
/** | ||
* | ||
* @return Returns the size of the chunk store on disk or service | ||
*/ | ||
public abstract long size(); | ||
|
||
/** | ||
* | ||
* @return The size of the chunkstore compressed | ||
*/ | ||
public abstract long compressedSize(); | ||
|
||
public abstract long maxSize(); | ||
|
||
public abstract void sync() throws IOException; | ||
|
||
/** | ||
* | ||
* @return bytes read since chunk store was started | ||
*/ | ||
public abstract long bytesRead(); | ||
|
||
/** | ||
* | ||
* @return returns bytes Written since chunk store was started | ||
*/ | ||
|
||
public abstract long bytesWritten(); | ||
|
||
/** | ||
* reserves a write position within the chunk store for later use. If | ||
* implemented but unused this should return 0 | ||
* | ||
* @param len | ||
* the length of the chunk that will be stored | ||
* @return the position with the chunk store where the chunk will be written | ||
* @throws IOException | ||
*/ | ||
// public abstract long reserveWritePosition(int len) throws IOException; | ||
|
||
/** | ||
* writes the chunk to the chunk store | ||
* | ||
* @param hash | ||
* the unique (md5 or SHA) hash for the data stored. | ||
* @param chunk | ||
* the actual data to be stored | ||
* @param len | ||
* the length of the data to be stored | ||
* @param start | ||
* the position within the chunk store that the chunk should be | ||
* stored at | ||
* @throws IOException | ||
*/ | ||
public abstract long writeChunk(byte[] hash, byte[] chunk, int len) | ||
throws IOException; | ||
|
||
/** | ||
* gets a chunk of data from the chunk store | ||
* | ||
* @param hash | ||
* hash the unique (md5 or SHA) hash for the data requested | ||
* @param start | ||
* the position within the chunk store that the chunk should be | ||
* located | ||
* @param len | ||
* the length of the data | ||
* @return the chunk of data requested | ||
* @throws IOException | ||
*/ | ||
public abstract byte[] getChunk(byte[] hash, long start, int len) | ||
throws IOException,DataArchivedException; | ||
|
||
/** | ||
* tunes the chunk store to a specific size | ||
* | ||
* @param length | ||
* the space that should be allocated on disk for this chunk | ||
* store. | ||
* @throws IOException | ||
*/ | ||
|
||
public abstract void deleteChunk(byte[] hash, long start, int len) | ||
throws IOException; | ||
|
||
public abstract void deleteDuplicate(byte[] hash, long start, int len) | ||
throws IOException; | ||
|
||
public abstract ChunkData getNextChunck() throws IOException; | ||
|
||
public abstract void iterationInit() throws IOException; | ||
|
||
package org.opendedup.sdfs.filestore; | ||
|
||
import java.io.IOException; | ||
|
||
import org.opendedup.collections.DataArchivedException; | ||
import org.w3c.dom.Element; | ||
|
||
/** | ||
* | ||
* @author Sam Silverberg The AbstractChunkStore is used as in interface to | ||
* store deduped chunks of data from clients to disk or some other | ||
* storage device or service for later retrieval. Within the chunk store | ||
* service itself data passes as follows | ||
* | ||
* client -> TCP -> Client Thread ->HashChunkService -> TCHashStore -> | ||
* AbstactChunkStore | ||
* | ||
* @see FileChunkStore | ||
* @see S3ChunkStore | ||
* @see NullChunkStore | ||
* @see TCChunkStore | ||
* | ||
*/ | ||
|
||
public interface AbstractChunkStore { | ||
|
||
public String restoreBlock(long id, byte[] hash) throws IOException; | ||
|
||
public boolean blockRestored(String id); | ||
|
||
public abstract long getFreeBlocks(); | ||
|
||
public abstract int getReadSpeed(); | ||
|
||
public abstract int getWriteSpeed(); | ||
|
||
public abstract long getCacheSize(); | ||
|
||
public abstract long getMaxCacheSize(); | ||
|
||
public abstract void setReadSpeed(int kbps); | ||
|
||
public abstract void setWriteSpeed(int kbps); | ||
|
||
public abstract void setCacheSize(long sz) throws IOException; | ||
|
||
/** | ||
* Closes the chunk store, if this is required | ||
*/ | ||
public abstract void close(); | ||
|
||
/** | ||
* Closes the chunk store, if this is required | ||
*/ | ||
public abstract void init(Element config) throws IOException; | ||
|
||
/** | ||
* Each chunk store must have a unique name. | ||
* | ||
* @return returns the unique name of the chunk store | ||
*/ | ||
public abstract String getName(); | ||
|
||
/** | ||
* Sets the name of the chunk store | ||
* | ||
* @param name | ||
* the name of the chunk store. | ||
*/ | ||
public abstract void setName(String name); | ||
|
||
/** | ||
* | ||
* @return Returns the size of the chunk store on disk or service | ||
*/ | ||
public abstract long size(); | ||
|
||
/** | ||
* | ||
* @return The size of the chunkstore compressed | ||
*/ | ||
public abstract long compressedSize(); | ||
|
||
public abstract long maxSize(); | ||
|
||
public abstract void sync() throws IOException; | ||
|
||
/** | ||
* | ||
* @return bytes read since chunk store was started | ||
*/ | ||
public abstract long bytesRead(); | ||
|
||
/** | ||
* | ||
* @return returns bytes Written since chunk store was started | ||
*/ | ||
|
||
public abstract long bytesWritten(); | ||
|
||
/** | ||
* reserves a write position within the chunk store for later use. If | ||
* implemented but unused this should return 0 | ||
* | ||
* @param len | ||
* the length of the chunk that will be stored | ||
* @return the position with the chunk store where the chunk will be written | ||
* @throws IOException | ||
*/ | ||
// public abstract long reserveWritePosition(int len) throws IOException; | ||
|
||
/** | ||
* writes the chunk to the chunk store | ||
* | ||
* @param hash | ||
* the unique (md5 or SHA) hash for the data stored. | ||
* @param chunk | ||
* the actual data to be stored | ||
* @param len | ||
* the length of the data to be stored | ||
* @param start | ||
* the position within the chunk store that the chunk should be | ||
* stored at | ||
* @throws IOException | ||
*/ | ||
public abstract long writeChunk(byte[] hash, byte[] chunk, int len) | ||
throws IOException; | ||
|
||
/** | ||
* gets a chunk of data from the chunk store | ||
* | ||
* @param hash | ||
* hash the unique (md5 or SHA) hash for the data requested | ||
* @param start | ||
* the position within the chunk store that the chunk should be | ||
* located | ||
* @param len | ||
* the length of the data | ||
* @return the chunk of data requested | ||
* @throws IOException | ||
*/ | ||
public abstract byte[] getChunk(byte[] hash, long start, int len) | ||
throws IOException,DataArchivedException; | ||
|
||
/** | ||
* tunes the chunk store to a specific size | ||
* | ||
* @param length | ||
* the space that should be allocated on disk for this chunk | ||
* store. | ||
* @throws IOException | ||
*/ | ||
|
||
public abstract void deleteChunk(byte[] hash, long start, int len) | ||
throws IOException; | ||
|
||
public abstract void deleteDuplicate(byte[] hash, long start, int len) | ||
throws IOException; | ||
|
||
public abstract ChunkData getNextChunck() throws IOException; | ||
|
||
public abstract void iterationInit(boolean getchunks) throws IOException; | ||
|
||
} |
Oops, something went wrong.