Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

S3 client config #24

Merged
merged 4 commits into from
Jun 5, 2024
Merged

S3 client config #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a669885
style: code style
bogovicj Jun 5, 2024
ee0006d
chore: bump n5-zarr
bogovicj Jun 5, 2024
836a63a
feat: N5Factory add s3ClientConfiguration
bogovicj Jun 5, 2024
6dcef76
chore: bump n5-aws-s3 4.2.0
bogovicj Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,11 @@
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

<n5.version>3.2.0</n5.version>
<n5-aws-s3.version>4.1.2</n5-aws-s3.version>
<n5-aws-s3.version>4.2.0</n5-aws-s3.version>
<n5-hdf5.version>2.2.0</n5-hdf5.version>
<n5-imglib2.version>7.0.0</n5-imglib2.version>
<n5-google-cloud.version>4.1.0</n5-google-cloud.version>
<n5-zarr.version>1.3.1</n5-zarr.version>
<n5-zarr.version>1.3.4</n5-zarr.version>

<jackson-jq.version>1.0.0-preview.20191208</jackson-jq.version>
<alphanumeric-comparator.version>1.4.1</alphanumeric-comparator.version>
Expand Down
73 changes: 42 additions & 31 deletions src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,19 @@
*/
package org.janelia.saalfeldlab.n5.universe;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.google.cloud.storage.Storage;
import com.google.gson.GsonBuilder;
import net.imglib2.util.Pair;
import net.imglib2.util.ValuePair;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;

import org.apache.commons.lang3.function.TriFunction;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils;
Expand All @@ -55,17 +61,15 @@
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader;
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter;

import javax.annotation.Nullable;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.google.cloud.storage.Storage;
import com.google.gson.GsonBuilder;

import net.imglib2.util.Pair;
import net.imglib2.util.ValuePair;

/**
* Factory for various N5 readers and writers. Implementation specific
Expand Down Expand Up @@ -94,6 +98,7 @@ public class N5Factory implements Serializable {
private String googleCloudProjectId = null;
private String s3Region = null;
private AWSCredentials s3Credentials = null;
private ClientConfiguration s3ClientConfiguration = null;
private boolean s3Anonymous = true;
private String s3Endpoint;

Expand Down Expand Up @@ -182,6 +187,12 @@ public N5Factory s3UseCredentials(final AWSCredentials credentials) {
return this;
}

public N5Factory s3ClientConfiguration(final ClientConfiguration clientConfiguration) {

this.s3ClientConfiguration = clientConfiguration;
return this;
}

@Deprecated
public N5Factory s3RetryWithCredentials() {

Expand All @@ -203,7 +214,7 @@ public N5Factory s3Region(final String s3Region) {
AmazonS3 createS3(final String uri) {

try {
return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region);
return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3ClientConfiguration, s3Region);
} catch (final Throwable e) {
throw new N5Exception("Could not create s3 client from uri: " + uri, e);
}
Expand All @@ -228,7 +239,7 @@ KeyValueAccess getKeyValueAccess(final URI uri) {
* backend take into account reasonable defaults when possible.
* Here we test from most to least restrictive.
* See the Javadoc for more details. */
for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) {
for (final KeyValueAccessBackend backend : KeyValueAccessBackend.values()) {
final KeyValueAccess kva = backend.apply(uri, this);
if (kva != null)
return kva;
Expand Down Expand Up @@ -352,10 +363,10 @@ public N5Reader openReader(final String uri) {
private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, String containerPath) {

if (storage == null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
try {
return openReader(format, access, containerPath);
} catch (Throwable e) {
} catch (final Throwable e) {
}
}
throw new N5Exception("Unable to open " + containerPath + " as N5Reader");
Expand Down Expand Up @@ -480,10 +491,10 @@ public N5Writer openWriter(final String uri) {
private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final String containerPath) {

if (storage == null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
try {
return openWriter(format, access, containerPath);
} catch (Throwable ignored) {
} catch (final Throwable ignored) {
}
}
throw new N5Exception("Unable to open " + containerPath + " as N5Writer");
Expand Down Expand Up @@ -511,7 +522,7 @@ private <T extends N5Reader> T openN5ContainerWithStorageFormat(
try {
final URI asUri = StorageFormat.parseUri(uri).getB();
return openWithFormat.apply(format, asUri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot create N5 Container (" + format + ") at " + uri, e);
}
}
Expand Down Expand Up @@ -547,7 +558,7 @@ private <T extends N5Reader> T openN5Container(
final Pair<StorageFormat, URI> storageAndUri;
try {
storageAndUri = StorageFormat.parseUri(containerUri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception("Unable to open " + containerUri + " as N5 Container", e);
}
final StorageFormat format = storageAndUri.getA();
Expand Down Expand Up @@ -631,7 +642,7 @@ public enum StorageFormat {

public static StorageFormat guessStorageFromUri(URI uri) {

for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
if (format.uriTest.test(uri))
return format;
}
Expand All @@ -656,7 +667,7 @@ public static Pair<StorageFormat, String> getStorageFromNestedScheme(String uri)
final String storageFormatScheme = storageSchemeMatcher.group(STORAGE_SCHEME_GROUP);
final String uriGroup = storageSchemeMatcher.group(URI_GROUP);
if (storageFormatScheme != null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
if (format.schemePattern.asPredicate().test(storageFormatScheme))
return new ValuePair<>(format, uriGroup);
}
Expand Down Expand Up @@ -690,15 +701,15 @@ public static N5Reader createReader(String containerUri) {
private static URI parseUriFromString(String uri) {
try {
return URI.create(uri);
} catch (Throwable ignore) {}
} catch (final Throwable ignore) {}

try {
return Paths.get(uri).toUri();
} catch (Throwable ignore) {}
} catch (final Throwable ignore) {}

try {
return N5URI.encodeAsUri(uri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception(e);
}
}
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/janelia/saalfeldlab/n5/universe/N5TreeNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ public N5TreeNode addPath( final String path, Function<String, N5TreeNode> const

// get the appropriate child along the path if it exists, otherwise add it
N5TreeNode child = null;
Stream<N5TreeNode> cs = children.stream().filter( n -> n.getNodeName().equals(childName));
Optional<N5TreeNode> copt = cs.findFirst();
final Stream<N5TreeNode> cs = children.stream().filter( n -> n.getNodeName().equals(childName));
final Optional<N5TreeNode> copt = cs.findFirst();
if( copt.isPresent() )
child = copt.get();
else {
Expand Down Expand Up @@ -199,8 +199,8 @@ public boolean structureEquals( N5TreeNode other )
return false;

boolean childrenEqual = true;
for( N5TreeNode c : childrenList()) {
Optional<N5TreeNode> otherChildOpt = other.childrenList().stream()
for( final N5TreeNode c : childrenList()) {
final Optional<N5TreeNode> otherChildOpt = other.childrenList().stream()
.filter( x -> x.getNodeName().equals( c.getNodeName()))
.findFirst();

Expand All @@ -221,9 +221,9 @@ public String printRecursive() {

private static String printRecursiveHelper(N5TreeNode node, String prefix) {

StringBuffer out = new StringBuffer();
final StringBuffer out = new StringBuffer();
out.append(prefix + node.path + "\n");
for (N5TreeNode c : node.childrenList()) {
for (final N5TreeNode c : node.childrenList()) {
System.out.println(c.path);
out.append(printRecursiveHelper(c, prefix + " "));
}
Expand Down
Loading