Do any languages / compilers utilize the x86 ENTER instruction with a nonzero nesting level?

Question

Those familiar with x86 assembly programming are very used to the typical function prologue / epilogue:

push ebp ; Save old frame pointer.
mov  ebp, esp ; Po         


        

Answer 1

As Iwillnotexist Idonotexist pointed out, GCC does support nested functions in C, using the exact syntax I've shown above.

However, it does not use ENTER instruction. Instead, variables which are used in nested functions are grouped together in the local variables area, and a pointer to this group is passed to the nested function. Interestingly, this "pointer to parent variables" is passed via a nonstandard mechanism: On x64 it is passed in r10, and on x86 (cdecl) it is passed in ecx, which is reserved for the this pointer in C++ (which doesn't support nested functions anyway).

#include 
void func_a(void)
{
    int a1 = 0x1001;
    int a2=2, a3=3, a4=4;
    int a5 = 0x1005;

    void func_b(int p1, int p2)
    {
        /* Use variables from func_a() */
        printf("a1=%d a5=%d\n", a1, a5);
    }
    func_b(1, 2);
}

int main(void)
{
    func_a();
    return 0;
}

Produces the following (snippet of) code when compiled for 64-bit:

00000000004004dc :
  4004dc:   push   rbp
  4004dd:   mov    rbp,rsp
  4004e0:   sub    rsp,0x10
  4004e4:   mov    DWORD PTR [rbp-0x4],edi
  4004e7:   mov    DWORD PTR [rbp-0x8],esi
  4004ea:   mov    rax,r10                    ; ptr to calling function "shared" vars
  4004ed:   mov    ecx,DWORD PTR [rax+0x4]
  4004f0:   mov    eax,DWORD PTR [rax]
  4004f2:   mov    edx,eax
  4004f4:   mov    esi,ecx
  4004f6:   mov    edi,0x400610
  4004fb:   mov    eax,0x0
  400500:   call   4003b0 
  400505:   leave  
  400506:   ret    

0000000000400507 :
  400507:   push   rbp
  400508:   mov    rbp,rsp
  40050b:   sub    rsp,0x20
  40050f:   mov    DWORD PTR [rbp-0x1c],0x1001
  400516:   mov    DWORD PTR [rbp-0x4],0x2
  40051d:   mov    DWORD PTR [rbp-0x8],0x3
  400524:   mov    DWORD PTR [rbp-0xc],0x4
  40052b:   mov    DWORD PTR [rbp-0x20],0x1005
  400532:   lea    rax,[rbp-0x20]              ; Pass a, b to the nested function
  400536:   mov    r10,rax                     ; in r10 !
  400539:   mov    esi,0x2
  40053e:   mov    edi,0x1
  400543:   call   4004dc 
  400548:   leave  
  400549:   ret  

_{Output from objdump --no-show-raw-insn -d -Mintel}

This would be equivalent to something more verbose like this:

struct func_a_ctx
{
    int a1, a5;
};

void func_b(struct func_a_ctx *ctx, int p1, int p2)
{
    /* Use variables from func_a() */
    printf("a1=%d a5=%d\n", ctx->a1, ctx->a5);
}

void func_a(void)
{
    int a2=2, a3=3, a4=4;
    struct func_a_ctx ctx = {
        .a1 = 0x1001,
        .a5 = 0x1005,
    };

    func_b(&ctx, 1, 2);
}