Use of GL_STENCIL_TEST to render concave polygons

Question

I\'m working on a custom geometry library adapted to Quartz Composer and trying to draw some concave polygons in a Plug In.

I implemented the poly2tri library, so t

Answer 1

...
glClearStencil(0);
...

Be aware that glClearStencil() just sets a bit of state and doesn't actually clear the stencil buffer.

Try adding a glClear( GL_STENCIL_BUFFER_BIT ) somewhere before each polygon.

EDIT: Like this:

#include <GL/glut.h>
#include <glm/glm.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <vector>

std::vector< glm::vec2 > pts;
bool leftHeld = true;
glm::vec2* dragPt = NULL;
void mouse( int button, int state, int x, int y )
{
    glm::vec2 pt( x, glutGet( GLUT_WINDOW_HEIGHT ) - y );

    // left mouse button starts dragging a point
    dragPt = NULL;
    leftHeld = false;
    if( button == GLUT_LEFT_BUTTON && state == GLUT_DOWN )
    {
        leftHeld = true;
        size_t minIdx = 0;
        for( size_t i = 0; i < pts.size(); ++i )
        {
            float newDist = glm::distance( pt, pts[ i ] );
            float oldDist = glm::distance( pt, pts[ minIdx ] );
            if( newDist <= oldDist && newDist < 15.0f )
            {
                minIdx = i;
                dragPt = &pts[ minIdx ];
            }
        }
    }

    // middle mouse button clears all points
    if( button == GLUT_MIDDLE_BUTTON && state == GLUT_UP )
    {
        pts.clear();
    }

    // right mouse button adds a point
    if( button == GLUT_RIGHT_BUTTON && state == GLUT_UP )
    {
        pts.push_back( pt );
    }

    glutPostRedisplay();
}

void motion( int x, int y )
{
    glm::vec2 pt( x, glutGet( GLUT_WINDOW_HEIGHT ) - y );
    if( dragPt && leftHeld )
    {
        *dragPt = pt;
        glutPostRedisplay();
    }
}

void glLine( const std::vector< glm::vec2 >& line, GLenum mode )
{
    glBegin( mode );
    for( size_t i = 0; i < line.size(); ++i )
    {
        glVertex2f( line[i].x, line[i].y );
    }
    glEnd();
}

void display()
{
    glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );

    glMatrixMode( GL_PROJECTION );
    glLoadIdentity();
    double w = glutGet( GLUT_WINDOW_WIDTH );
    double h = glutGet( GLUT_WINDOW_HEIGHT );
    glOrtho( 0, w, 0, h, -1, 1 );

    glMatrixMode( GL_MODELVIEW );
    glLoadIdentity();

    // draw polygon
    glClear( GL_STENCIL_BUFFER_BIT );
    {
        // fill stencil buffer
        glEnable( GL_STENCIL_TEST );
        glColorMask( GL_FALSE,GL_FALSE,GL_FALSE,GL_FALSE );
        glStencilOp( GL_KEEP, GL_KEEP, GL_INVERT );
        glStencilFunc( GL_ALWAYS, 0x1, 0x1 );
        glBegin( GL_TRIANGLES );
        for( size_t i = 1; i+1 < pts.size(); ++i )
        {
            glVertex2fv( glm::value_ptr( pts[ 0 ] ) );
            glVertex2fv( glm::value_ptr( pts[ i ] ) );
            glVertex2fv( glm::value_ptr( pts[ i+1 ] ) );
        }
        glEnd();

        // fill color buffer
        glColor3ub( 0, 128, 0 );
        glColorMask( GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE );
        glStencilFunc( GL_EQUAL, 0x1, 0x1 );
        glBegin( GL_TRIANGLES );
        for( size_t i = 1; i+1 < pts.size(); ++i )
        {
            glVertex2fv( glm::value_ptr( pts[ 0 ] ) );
            glVertex2fv( glm::value_ptr( pts[ i ] ) );
            glVertex2fv( glm::value_ptr( pts[ i+1 ] ) );
        }
        glEnd();
        glDisable( GL_STENCIL_TEST );
    }

    // draw polygon boundary
    glLineWidth( 1 );
    glColor3ub( 255, 255, 255 );
    glLine( pts, GL_LINE_LOOP );

    // draw vertexes
    glPointSize( 9 );
    glColor3ub( 255, 0, 0 );
    glLine( pts, GL_POINTS );

    glutSwapBuffers();
}

int main( int argc, char **argv )
{
    glutInit( &argc, argv );
    glutInitDisplayMode( GLUT_RGBA | GLUT_DEPTH | GLUT_DOUBLE | GLUT_STENCIL );
    glutInitWindowSize( 640, 480 );
    glutCreateWindow( "GLUT" );
    glutMouseFunc( mouse );
    glutMotionFunc( motion );
    glutDisplayFunc( display );
    glutMainLoop();
    return 0;
}