How to decrypt file in Java encrypted with openssl command using AES?

Question

I need to decrypt in JAVA a file encrypted in UNIX with the following command:

openssl aes-256-cbc -a -salt -in password.txt -out password.txt.enc
mypass
myp         


        

Answer 1

Below are OpenSSLPBEInputStream and OpenSSLPBEOutputStream which can be used to encrypt/decrypt arbitrary streams of bytes in a way that is compatible with OpenSSL.

Example usage:

    // The original clear text bytes
    byte[] originalBytes = ...

    // Encrypt these bytes
    char[] pwd = "thePassword".toCharArray();
    ByteArrayOutputStream byteOS = new ByteArrayOutputStream();
    OpenSSLPBEOutputStream encOS = new OpenSSLPBEOutputStream(byteOS, ALGORITHM, 1, pwd);
    encOS.write(originalBytes);
    encOS.flush();
    byte[] encryptedBytes = byteOS.toByteArray();

    // Decrypt the bytes
    ByteArrayInputStream byteIS = new ByteArrayInputStream(encryptedBytes);
    OpenSSLPBEInputStream encIS = new OpenSSLPBEInputStream(byteIS, ALGORITHM, 1, pwd);

Where ALGORITHM (using just JDK classes) can be: "PBEWithMD5AndDES", "PBEWithMD5AndTripleDES", "PBEWithSHA1AndDESede", "PBEWithSHA1AndRC2_40".

To handle "openssl aes-256-cbc -a -salt -in password.txt -out password.txt.enc" of the original poster, add bouncey castle to the classpath, and use algorthm= "PBEWITHMD5AND256BITAES-CBC-OPENSSL".

/* Add BC provider, and fail fast if BC provider is not in classpath for some reason */
Security.addProvider(new BouncyCastleProvider());

The dependency:

    
        org.bouncycastle
        bcprov-jdk16
        1.44
    

The input stream:

import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;

public class OpenSSLPBEInputStream extends InputStream {

    private final static int READ_BLOCK_SIZE = 64 * 1024;

    private final Cipher cipher;
    private final InputStream inStream;
    private final byte[] bufferCipher = new byte[READ_BLOCK_SIZE];

    private byte[] bufferClear = null;

    private int index = Integer.MAX_VALUE;
    private int maxIndex = 0;

    public OpenSSLPBEInputStream(final InputStream streamIn, String algIn, int iterationCount, char[] password)
            throws IOException {
        this.inStream = streamIn;
        try {
            byte[] salt = readSalt();
            cipher = OpenSSLPBECommon.initializeCipher(password, salt, Cipher.DECRYPT_MODE, algIn, iterationCount);
        } catch (InvalidKeySpecException | NoSuchPaddingException | NoSuchAlgorithmException | InvalidKeyException | InvalidAlgorithmParameterException e) {
            throw new IOException(e);
        }
    }

    @Override
    public int available() throws IOException {
        return inStream.available();
    }

    @Override
    public int read() throws IOException {

        if (index > maxIndex) {
            index = 0;
            int read = inStream.read(bufferCipher);
            if (read != -1) {
                bufferClear = cipher.update(bufferCipher, 0, read);
            }
            if (read == -1 || bufferClear == null || bufferClear.length == 0) {
                try {
                    bufferClear = cipher.doFinal();
                } catch (IllegalBlockSizeException | BadPaddingException e) {
                    bufferClear = null;
                }
            }
            if (bufferClear == null || bufferClear.length == 0) {
                return -1;
            }
            maxIndex = bufferClear.length - 1;
        }
        return bufferClear[index++] & 0xff;

    }

    private byte[] readSalt() throws IOException {

        byte[] headerBytes = new byte[OpenSSLPBECommon.OPENSSL_HEADER_STRING.length()];
        inStream.read(headerBytes);
        String headerString = new String(headerBytes, OpenSSLPBECommon.OPENSSL_HEADER_ENCODE);

        if (!OpenSSLPBECommon.OPENSSL_HEADER_STRING.equals(headerString)) {
            throw new IOException("unexpected file header " + headerString);
        }

        byte[] salt = new byte[OpenSSLPBECommon.SALT_SIZE_BYTES];
        inStream.read(salt);

        return salt;
    }

}

The output stream:

import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import java.io.IOException;
import java.io.OutputStream;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.InvalidKeySpecException;

public class OpenSSLPBEOutputStream extends OutputStream {

private static final int BUFFER_SIZE = 5 * 1024 * 1024;

private final Cipher cipher;
private final OutputStream outStream;
private final byte[] buffer = new byte[BUFFER_SIZE];
private int bufferIndex = 0;

public OpenSSLPBEOutputStream(final OutputStream outputStream, String algIn, int iterationCount,
                              char[] password) throws IOException {
    outStream = outputStream;
    try {
        /* Create and use a random SALT for each instance of this output stream. */
        byte[] salt = new byte[PBECommon.SALT_SIZE_BYTES];
        new SecureRandom().nextBytes(salt);
        cipher = OpenSSLPBECommon.initializeCipher(password, salt, Cipher.ENCRYPT_MODE, algIn, iterationCount);
        /* Write header */
        writeHeader(salt);
    } catch (InvalidKeySpecException | NoSuchPaddingException | NoSuchAlgorithmException | InvalidKeyException | InvalidAlgorithmParameterException e) {
        throw new IOException(e);
    }
}

@Override
public void write(int b) throws IOException {
    buffer[bufferIndex] = (byte) b;
    bufferIndex++;
    if (bufferIndex == BUFFER_SIZE) {
        byte[] result = cipher.update(buffer, 0, bufferIndex);
        outStream.write(result);
        bufferIndex = 0;
    }
}

@Override
public void flush() throws IOException {
    if (bufferIndex > 0) {
        byte[] result;
        try {
            result = cipher.doFinal(buffer, 0, bufferIndex);
            outStream.write(result);
        } catch (IllegalBlockSizeException | BadPaddingException e) {
            throw new IOException(e);
        }
        bufferIndex = 0;
    }
}

@Override
public void close() throws IOException {
    flush();
    outStream.close();
}

private void writeHeader(byte[] salt) throws IOException {
    outStream.write(OpenSSLPBECommon.OPENSSL_HEADER_STRING.getBytes(OpenSSLPBECommon.OPENSSL_HEADER_ENCODE));
    outStream.write(salt);
}

}

Small common class:

import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.PBEParameterSpec;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;

class OpenSSLPBECommon {

protected static final int SALT_SIZE_BYTES = 8;
protected static final String OPENSSL_HEADER_STRING = "Salted__";
protected static final String OPENSSL_HEADER_ENCODE = "ASCII";

protected static Cipher initializeCipher(char[] password, byte[] salt, int cipherMode,
                                         final String algorithm, int iterationCount) throws NoSuchAlgorithmException, InvalidKeySpecException,
        InvalidKeyException, NoSuchPaddingException, InvalidAlgorithmParameterException {

    PBEKeySpec keySpec = new PBEKeySpec(password);
    SecretKeyFactory factory = SecretKeyFactory.getInstance(algorithm);
    SecretKey key = factory.generateSecret(keySpec);

    Cipher cipher = Cipher.getInstance(algorithm);
    cipher.init(cipherMode, key, new PBEParameterSpec(salt, iterationCount));

    return cipher;
}

}