Different signatures when using C routines and openssl dgst, rsautl commands

Question

I am using following statement to create a RSA public and private key.

openssl genrsa -out ksign_private.pem 1024 openssl rsa -in ksign_private.pem -pubout > ksign_public.pem

Then I have program that uses, PEM_read_RSAPrivateKey, EVP_PKEY_assign_RSA, EVP_SignInit, EVP_SignUpdate, EVP_SignFinal functions from openssl libcrypto to generate signature file.

I also have routine that verifies that signature can be verified using PEM_read_RSA_PUBKEY, EVP_PKEY_assign_RSA, EVP_VerifyInit, EVP_VerifyUpdate, EVP_VerifyFinal. Source code for these routines is attached below.

When using these functions I can create SHA1 signature, encrypt it with private key, and decrypt it using public key.

However I tried to use the same data file, same private, public key using the openssl rsautl and the signature that is getting created by openssl rsautl is vastly different.

openssl dgst -sha1 -binary < myData > testfile.sha1
openssl rsautl -sign -in testfile.sha1 -inkey ksign_private.pem -keyform PEM -out testfile.sig

Can any one tell me what options I am using wrong when using openssl rsautl or dgst command?

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <ctype.h>
#include <unistd.h>
#include <string.h>
#include <openssl/sha.h>
#include <errno.h>
#include <getopt.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>

int ksignEvpSign(FILE * private_key, FILE * inFileFP, FILE * outFileFP);
int ksignEvpVerify(FILE * public_key, FILE * dataFileFP, FILE * signFileFP);

int ksignEvpSign(FILE * privateKeyFP, FILE * inFileFP, FILE * outFileFP)
{
    RSA *rsa_pkey = NULL;
    EVP_PKEY *pkey = EVP_PKEY_new();
    EVP_MD_CTX ctx;
    unsigned char buffer[4096];
    size_t len;
    unsigned char *sig;
    unsigned int siglen;

    if (!PEM_read_RSAPrivateKey(privateKeyFP, &rsa_pkey, NULL, NULL)) {
        fprintf(stderr, "Error loading RSA Private Key File.\n");
        return 2;
    }

    if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey)) {
        fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
        return 3;
    }

    EVP_MD_CTX_init(&ctx);

    if (!EVP_SignInit(&ctx, EVP_sha1())) {
        fprintf(stderr, "EVP_SignInit: failed.\n");
        EVP_PKEY_free(pkey);
        return 3;
    }

    while ((len = fread(buffer, 1, sizeof buffer, inFileFP)) > 0) {
        if (!EVP_SignUpdate(&ctx, buffer, len)) {
            fprintf(stderr, "EVP_SignUpdate: failed.\n");
            EVP_PKEY_free(pkey);
            return 3;
        }
    }

    if (ferror(inFileFP)) {
        perror("input file");
        EVP_PKEY_free(pkey);
        return 4;
    }

    sig = malloc(EVP_PKEY_size(pkey));
    if (!EVP_SignFinal(&ctx, sig, &siglen, pkey)) {
        fprintf(stderr, "EVP_SignFinal: failed.\n");
        free(sig);
        EVP_PKEY_free(pkey);
        return 3;
    }
    fwrite(sig, siglen, 1, outFileFP);
    free(sig);
    EVP_PKEY_free(pkey);
    return 0;
}

int ksignEvpVerify(FILE * publicKeyFP, FILE * dataFileFP, FILE * sigFileFP)
{
    RSA *rsa_pkey = NULL;
    EVP_PKEY *pkey;
    EVP_MD_CTX ctx;
    unsigned char buffer[4096];
    size_t len;
    unsigned char *sig;
    unsigned int siglen;
    struct stat stat_buf;

    if (!PEM_read_RSA_PUBKEY(publicKeyFP, &rsa_pkey, NULL, NULL)) {
        fprintf(stderr, "Error loading RSA public Key File.\n");
        return 2;
    }
    pkey = EVP_PKEY_new();

    if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey)) {
        fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
        return 3;
    }
    /* Read the signature */
    if (fstat(fileno(sigFileFP), &stat_buf) == -1) {
        fprintf(stderr, "Unable to read signature \n");
        return 4;
    }
    siglen = stat_buf.st_size;
    sig = (unsigned char *)malloc(siglen);
    if (sig == NULL) {
        fprintf(stderr, "Unable to allocated %d bytes for signature\n",
            siglen);
        return 5;
    }
    if ((fread(sig, 1, siglen, sigFileFP)) != siglen) {
        fprintf(stderr, "Unable to read %d bytes for signature\n",
            siglen);
        return 6;
    }
/*
    printf("Signature:");
    for (i = 0; i < siglen; i++) {
        fprintf(stdout, "%02x", sig[i]);
        if (i % 16 == 15)
            fprintf(stdout, "\n");
    }
    fprintf(stdout, "\n");
*/

    EVP_MD_CTX_init(&ctx);

    if (!EVP_VerifyInit(&ctx, EVP_sha1())) {
        fprintf(stderr, "EVP_SignInit: failed.\n");
        EVP_PKEY_free(pkey);
        return 7;
    }

    while ((len = fread(buffer, 1, sizeof buffer, dataFileFP)) > 0) {
        if (!EVP_VerifyUpdate(&ctx, buffer, len)) {
            fprintf(stderr, "EVP_SignUpdate: failed.\n");
            EVP_PKEY_free(pkey);
            return 8;
        }
    }

    if (ferror(dataFileFP)) {
        perror("input file");
        EVP_PKEY_free(pkey);
        return 9;
    }

    if (!EVP_VerifyFinal(&ctx, sig, siglen, pkey)) {
        fprintf(stderr, "EVP_VerifyFinal: failed.\n");
        free(sig);
        EVP_PKEY_free(pkey);
        return 10;
    }
    free(sig);
    EVP_PKEY_free(pkey);
    return 0;
}

Answer 1

The pkeyutl command should be preferred to rsautl since pkeyutl can handle any algorithm. To obtain the same signature on the command line, you should use the following:

openssl pkeyutl -sign -in testfile.sha1 -inkey ksign_private.pem -pkeyopt digest:sha1 -outfile testfile.sig

The important part is telling openssl that you're using a digest value. Otherwise it seems to be signing a digest of your digest.

Answer 2

You can directly use dgst command to hash and sign data like:

openssl dgst -sha1 -binary -sign privkey.pem < myData > mySignature

see docs for all options.