Skip to content

Commit

Permalink
Merge pull request #8 from takipi/develop
Browse files Browse the repository at this point in the history
Releasing 1.6.0
  • Loading branch information
chook committed May 30, 2016
2 parents d672728 + 69e7278 commit 42de1b1
Show file tree
Hide file tree
Showing 29 changed files with 582 additions and 174 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ Build and run:
- clone the repo
- `cd takipi-storage`
- `mvn compile package`
- `java -jar target/takipi-storage-1.5.0.jar server settings.yml`
- `java -jar target/takipi-storage-1.6.0.jar server settings.yml`

Deploy:
- `wget https://s3.amazonaws.com/app-takipi-com/deploy/takipi-storage/takipi-storage-1.5.0.tar.gz`
- `wget https://s3.amazonaws.com/app-takipi-com/deploy/takipi-storage/takipi-storage-1.6.0.tar.gz`
- **Now with sudo**:
- `cd /opt`
- `tar zxvf <path-to-download>/takipi-storage-1.5.0.tar.gz`
- `tar zxvf <path-to-download>/takipi-storage-1.6.0.tar.gz`
- `cp /opt/takipi-storage/etc/takipi-storage /etc/init.d`
- Ubuntu: `/usr/sbin/update-rc.d takipi-storage defaults`
- RHEL: `/sbin/chkconfig takipi-storage on`
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

<groupId>com.takipi</groupId>
<artifactId>takipi-storage</artifactId>
<version>1.5.0</version>
<version>1.6.0</version>

<properties>
<!-- use UTF-8 for everything -->
Expand Down
38 changes: 18 additions & 20 deletions src/main/java/com/takipi/oss/storage/TakipiStorageMain.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@

import org.eclipse.jetty.servlets.CrossOriginFilter;

import com.takipi.oss.storage.fs.api.Filesystem;
import com.takipi.oss.storage.fs.folder.HashSubfolderFilesystem;
import com.takipi.oss.storage.health.FilesystemHealthCheck;
import com.takipi.oss.storage.resources.BinaryStorageResource;
import com.takipi.oss.storage.resources.JsonMultiDeleteStorageResource;
import com.takipi.oss.storage.resources.JsonMultiFetchStorageResource;
import com.takipi.oss.storage.resources.PingStorageResource;
import com.takipi.oss.storage.resources.StatusStorageResource;
import com.takipi.oss.storage.resources.TreeStorageResource;
import com.takipi.oss.storage.resources.diag.PingStorageResource;
import com.takipi.oss.storage.resources.diag.StatusStorageResource;
import com.takipi.oss.storage.resources.diag.TreeStorageResource;
import com.takipi.oss.storage.resources.fs.BinaryStorageResource;
import com.takipi.oss.storage.resources.fs.JsonMultiDeleteStorageResource;
import com.takipi.oss.storage.resources.fs.JsonMultiFetchStorageResource;
import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource;
import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource;

public class TakipiStorageMain extends Application<TakipiStorageConfiguration> {
public static void main(String[] args) throws Exception {
Expand All @@ -41,21 +41,19 @@ public void run(TakipiStorageConfiguration configuration, Environment environmen
if (configuration.isEnableCors()) {
enableCors(configuration, environment);
}

String folderPath = configuration.getFolderPath();

Filesystem fs = new HashSubfolderFilesystem(folderPath,
configuration.getMaxUsedStoragePercentage());

environment.jersey().register(new BinaryStorageResource(fs));
environment.jersey().register(new JsonMultiFetchStorageResource(fs));
environment.jersey().register(new JsonMultiDeleteStorageResource(fs));

environment.jersey().register(new BinaryStorageResource(configuration));
environment.jersey().register(new JsonMultiFetchStorageResource(configuration));
environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));

environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));

environment.jersey().register(new PingStorageResource());
environment.jersey().register(new TreeStorageResource(folderPath));
environment.jersey().register(new StatusStorageResource(folderPath));
environment.jersey().register(new TreeStorageResource(configuration));
environment.jersey().register(new StatusStorageResource(configuration));

environment.healthChecks().register("filesystem", new FilesystemHealthCheck(fs));
environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration));
}

private void enableCors(TakipiStorageConfiguration configuration, Environment environment) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package com.takipi.oss.storage.data.simple;

import com.takipi.oss.storage.data.EncodingType;

public class SimpleFetchRequest {
public EncodingType encodingType;
public String path;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.takipi.oss.storage.data.simple;

public class SimpleFetchResponse {
String data;

public SimpleFetchResponse(String data) {
this.data = data;
}

public String getData() {
return data;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.takipi.oss.storage.data.simple;

import com.takipi.oss.storage.data.EncodingType;

public class SimpleSearchRequest {
public EncodingType encodingType;
public String name;
public String baseSearchPath;
public boolean preventDuplicates;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.takipi.oss.storage.data.simple;

public class SimpleSearchResponse {
String data;
String path;

public SimpleSearchResponse(String data, String path) {
this.data = data;
this.path = path;
}

public String getData() {
return data;
}

public String getPath() {
return path;
}
}
22 changes: 6 additions & 16 deletions src/main/java/com/takipi/oss/storage/fs/api/Filesystem.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
import java.io.IOException;
import java.io.InputStream;

import com.takipi.oss.storage.fs.Record;

public interface Filesystem {
public interface Filesystem<T> extends FilesystemHealth {
/**
* Put record
*
Expand All @@ -16,7 +14,7 @@ public interface Filesystem {
* @throws IOException
* - if there's an error
*/
void put(Record record, InputStream is) throws IOException;
void put(T record, InputStream is) throws IOException;

/**
* Get record
Expand All @@ -27,7 +25,7 @@ public interface Filesystem {
* @throws IOException
* - if there's an error
*/
InputStream get(Record record) throws IOException;
InputStream get(T record) throws IOException;

/**
* Removes record from filesystem
Expand All @@ -37,7 +35,7 @@ public interface Filesystem {
* @throws IOException
* - if there's an error
*/
void delete(Record record) throws IOException;
void delete(T record) throws IOException;

/**
* True if record exists in filesystem
Expand All @@ -47,7 +45,7 @@ public interface Filesystem {
* @throws IOException
* - if there's an error
*/
boolean exists(Record record) throws IOException;
boolean exists(T record) throws IOException;

/**
* Returns the size of a record in the filesystem
Expand All @@ -57,13 +55,5 @@ public interface Filesystem {
* @throws IOException
* - if there's an error
*/
long size(Record record) throws IOException;

/**
* Checks if the filesystem is healthy. This will be called during health
* check
*
* @return true iff the filesystem is healthy
*/
boolean healthy();
long size(T record) throws IOException;
}
11 changes: 11 additions & 0 deletions src/main/java/com/takipi/oss/storage/fs/api/FilesystemHealth.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package com.takipi.oss.storage.fs.api;

public interface FilesystemHealth {
/**
* Checks if the filesystem is healthy. This will be called during health
* check
*
* @return true iff the filesystem is healthy
*/
boolean healthy();
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,50 +7,29 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.commons.io.IOUtils;

import com.takipi.oss.storage.fs.Record;
import com.takipi.oss.storage.fs.api.Filesystem;

public class FolderFilesystem implements Filesystem {
private final File root;
private final double maxUsedStoragePercentage;

public abstract class FolderFilesystem<T> extends FolderFilesystemHealth implements Filesystem<T> {
public FolderFilesystem(String rootFolder, double maxUsedStoragePercentage) {
this.root = new File(rootFolder);
this.maxUsedStoragePercentage = maxUsedStoragePercentage;

if (!healthy()) {
throw new IllegalStateException("Problem with path " + rootFolder);
}
super(rootFolder, maxUsedStoragePercentage);
}

@Override
public boolean healthy() {
return (folderCheck() && maxUsedStorageCheck());
}

private boolean folderCheck() {
return ((this.root.canRead()) && (this.root.canWrite()));
}

private boolean maxUsedStorageCheck() {
return ((maxUsedStoragePercentage >= 0) && (maxUsedStoragePercentage < 1) && ((this.root.getUsableSpace() / this.root
.getTotalSpace()) <= maxUsedStoragePercentage));
public File getRoot() {
return root;
}

@Override
public InputStream get(Record record) throws IOException {
public InputStream get(T record) throws IOException {
File file = new File(buildPath(record));

return new FileInputStream(file);
}

@Override
public void put(Record record, InputStream is) throws IOException {
public void put(T record, InputStream is) throws IOException {
File file = new File(buildPath(record));

beforePut(file);
Expand All @@ -64,7 +43,7 @@ public void put(Record record, InputStream is) throws IOException {
}

@Override
public void delete(Record record) throws IOException {
public void delete(T record) throws IOException {
File file = new File(buildPath(record));

if (file.exists() && file.canWrite()) {
Expand All @@ -79,7 +58,7 @@ public void delete(Record record) throws IOException {
}

@Override
public boolean exists(Record record) throws IOException {
public boolean exists(T record) throws IOException {
File file = new File(buildPath(record));

if (file.exists() && file.canRead()) {
Expand All @@ -90,7 +69,7 @@ public boolean exists(Record record) throws IOException {
}

@Override
public long size(Record record) throws IOException {
public long size(T record) throws IOException {
File file = new File(buildPath(record));

if (file.exists() && file.canRead()) {
Expand All @@ -99,25 +78,10 @@ public long size(Record record) throws IOException {

throw new FileNotFoundException();
}

protected String buildPath(Record record) {
Path recordPath = Paths.get(root.getPath(), escape(record.getServiceId()), escape(record.getType()),
escape(record.getServiceId()));

return recordPath.toString();
}

protected String buildPath(String key) {
Path recordPath = Paths.get(root.getPath(), escape(key));

return recordPath.toString();
}


protected void beforePut(File file) {
file.getParentFile().mkdirs();
}

protected String escape(String value) {
return value.replace("..", "__").replace("/", "-").replace("\\", "-");
}

protected abstract String buildPath(T record);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package com.takipi.oss.storage.fs.folder;

import java.io.File;

import com.takipi.oss.storage.fs.api.FilesystemHealth;

public class FolderFilesystemHealth implements FilesystemHealth {
protected final File root;
private final double maxUsedStoragePercentage;

public FolderFilesystemHealth(String rootFolder, double maxUsedStoragePercentage) {
this.root = new File(rootFolder);
this.maxUsedStoragePercentage = maxUsedStoragePercentage;

if (!healthy()) {
throw new IllegalStateException("Problem with path " + rootFolder);
}
}

@Override
public boolean healthy() {
return (folderCheck() && maxUsedStorageCheck());
}

private boolean folderCheck() {
return ((this.root.canRead()) && (this.root.canWrite()));
}

private boolean maxUsedStorageCheck() {
return ((maxUsedStoragePercentage >= 0) && (maxUsedStoragePercentage < 1) && ((this.root.getUsableSpace() / this.root
.getTotalSpace()) <= maxUsedStoragePercentage));
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.takipi.oss.storage.fs.folder;
package com.takipi.oss.storage.fs.folder.record;

import java.io.File;
import java.nio.file.Path;
Expand All @@ -9,7 +9,7 @@
import com.google.common.hash.Hashing;
import com.takipi.oss.storage.fs.Record;

public class HashSubfolderFilesystem extends FolderFilesystem {
public class HashSubfolderFilesystem extends RecordFilesystem {
private HashFunction func;

public HashSubfolderFilesystem(String rootFolder, double maxUsedStoragePercentage) {
Expand All @@ -21,12 +21,10 @@ public HashSubfolderFilesystem(String rootFolder, double maxUsedStoragePercentag
@Override
protected String buildPath(Record record) {
String key = record.getKey();
String path = super.buildPath(key);
File pathParent = new File(path).getParentFile();

String hashKey = hashKey(key);

Path recordPath = Paths.get(pathParent.getPath(), escape(record.getServiceId()), escape(record.getType()),
Path recordPath = Paths.get(root.getPath(), escape(record.getServiceId()), escape(record.getType()),
hashKey, escape(key));

return recordPath.toString();
Expand Down
Loading

0 comments on commit 42de1b1

Please sign in to comment.