How to calculate the normal matrix?

Question

I have some trouble with my normal matrix.

vs.glsl

#version 440

in vec3 vPosition;
in vec3 vNormal;

out vec4 eyeCordFs;
out vec4 eyeNormalFs;

uniform mat4 model;
uniform mat4 view;
uniform mat4 proj;

void main()
{   
    mat4 modelView = view * model;
    mat4 normalMatrix = view * transpose(inverse(model));
    vec4 eyeNorm = normalize(normalMatrix * vec4(vNormal, 0.0));
    vec4 eyeCord= modelView * vec4(vPosition, 1.0);

    eyeCordFs = eyeCord;
    eyeNormalFs = eyeNorm;

    gl_Position = proj * modelView * vec4( vPosition,1.0);
}

fs.glsl

#version 440

in vec4 eyeCordFs;
in vec4 eyeNormalFs;

out vec3 outputColor;

uniform vec4 lightPos;

void main()
{
    vec4 s = normalize(lightPos - eyeCordFs) ;
    vec4 r = reflect(-s,eyeNormalFs);
    vec4 v = normalize(-eyeCordFs);
    float spec = max( dot(v,r),0.0 );
    float diff = max(dot(eyeNormalFs,s),0.0);

    vec3 diffColor = diff * vec3(1,0,0);
    vec3 specColor = pow(spec,3) * vec3(1,1,1);
    vec3 ambientColor = vec3(0.1,0.1,0.1);

    outputColor =  diffColor + 0.5 * specColor + ambientColor; 
}

The output of this looks like

Which seemed a bit strange to me. But I knew that I am not scaling anything so I thought I could use the modelView matrix to transform my normals.

So I changed the line

vec4 eyeNorm = normalize(normalMatrix * vec4(vNormal, 0.0));

to

vec4 eyeNorm = normalize(modelView * vec4(vNormal, 0.0));

And the output now looks like this

which looks correct. Am I calculating my normalMatrix the wrong way?

Answer 1

The normal matrix is the transpose inverse of the modelview matrix. So in GLSL it would be

mat4 normalMatrix = transpose(inverse(modelView));

However you should NOT calculate the normal matrix in the shader. Doing that in the shader wastes a lot of precious GPU cycles. Matrix inversion is not a cheap operation to start with and doing it in the shader forces the GPU to perform the calculation for each and every vertex again and again. Precalculate it on the CPU and pass it as a uniform.