How to list all AWS S3 objects in a bucket using Java

Question

What is the simplest way to get a list of all items within an S3 bucket using Java?

List s3objects = s3.listObjects(bucketName,prefix)         


        

Answer 1

You don't want to list all 1000 object in your bucket at a time. A more robust solution will be to fetch a max of 10 objects at a time. You can do this with the withMaxKeys method.

The following code creates an S3 client, fetches 10 or less objects at a time and filters based on a prefix and generates a pre-signed url for the fetched object:

import com.amazonaws.HttpMethod;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;

import java.net.URL;
import java.util.Date;

/**
 * @author shabab
 * @since 21 Sep, 2020
 */
public class AwsMain {

    static final String ACCESS_KEY = "";
    static final String SECRET = "";
    static final Regions BUCKET_REGION = Regions.DEFAULT_REGION;
    static final String BUCKET_NAME = "";

    public static void main(String[] args) {
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(ACCESS_KEY, SECRET);

        try {
            final AmazonS3 s3Client = AmazonS3ClientBuilder
                    .standard()
                    .withRegion(BUCKET_REGION)
                    .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
                    .build();

            ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(BUCKET_NAME).withMaxKeys(10);
            ListObjectsV2Result result;

            do {
                result = s3Client.listObjectsV2(req);

                result.getObjectSummaries()
                        .stream()
                        .filter(s3ObjectSummary -> {
                            return s3ObjectSummary.getKey().contains("Market-subscriptions/")
                                    && !s3ObjectSummary.getKey().equals("Market-subscriptions/");
                        })
                        .forEach(s3ObjectSummary -> {

                            GeneratePresignedUrlRequest generatePresignedUrlRequest =
                                    new GeneratePresignedUrlRequest(BUCKET_NAME, s3ObjectSummary.getKey())
                                            .withMethod(HttpMethod.GET)
                                            .withExpiration(getExpirationDate());

                            URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);

                            System.out.println(s3ObjectSummary.getKey() + " Pre-Signed URL: " + url.toString());
                        });

                String token = result.getNextContinuationToken();
                req.setContinuationToken(token);

            } while (result.isTruncated());
        } catch (SdkClientException e) {
            e.printStackTrace();
        }

    }

    private static Date getExpirationDate() {
        Date expiration = new java.util.Date();
        long expTimeMillis = expiration.getTime();
        expTimeMillis += 1000 * 60 * 60;
        expiration.setTime(expTimeMillis);

        return expiration;
    }
}