Is there a simple example of using antlr4 to create an AST from java source code and extract methods, variables and comments? [closed]

Answer 1

here is a detail example, (borrows from https://github.com/satnam-sandhu/ASTGenerator), i do some change for getting line number.

helloworld.java

public class HelloWorld {
public static void main(String[] args) {
    System.out.println("Hello, World");
    }
}

JavaAstGeneratorDOT.java

import antlr.Java8Lexer;
import antlr.Java8Parser;

import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.misc.Interval;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.ArrayList;


public class JavaAstGeneratorDOT {

    static ArrayList LineNum = new ArrayList();
    static ArrayList Type = new ArrayList();
    static ArrayList Content = new ArrayList();
    static ArrayList RawLineNum = new ArrayList();

    private static String readFile(String pathname) throws IOException {
        File file = new File(pathname);
        byte[] encoded = Files.readAllBytes(file.toPath());
        return new String(encoded, Charset.forName("UTF-8"));
    }

    public static void main(String args[]) throws IOException {
        String path = "helloworld.java";
        String inputString = readFile(path);
        ANTLRInputStream input = new ANTLRInputStream(inputString);
        Java8Lexer lexer = new Java8Lexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        Java8Parser parser = new Java8Parser(tokens);
        ParserRuleContext ctx = parser.compilationUnit();
//      ParserRuleContext ctx = parser.statementExpressionList();
//      ParserRuleContext ctx = parser.methodDeclaration();

        generateAST(ctx, false, 0, tokens);
        String filename = path.substring(path.lastIndexOf("\\") + 1, path.lastIndexOf("."));
        String save_dot_filename = String.format("ast_%s.dot", filename);
        PrintWriter writer = new PrintWriter(save_dot_filename);
        writer.println(String.format("digraph %s {", filename));
        printDOT(writer);
        writer.println("}");
        writer.close();
    }

    private static void generateAST(RuleContext ctx, boolean verbose, int indentation, CommonTokenStream tokens) {
        boolean toBeIgnored = !verbose && ctx.getChildCount() == 1 && ctx.getChild(0) instanceof ParserRuleContext;
        if (!toBeIgnored) {
            String ruleName = Java8Parser.ruleNames[ctx.getRuleIndex()];
            LineNum.add(Integer.toString(indentation));
            Type.add(ruleName);
            Content.add(ctx.getText());

            // get line number, added by tsmc.sumihui, 20190425
            Interval sourceInterval = ctx.getSourceInterval();
            Token firstToken = tokens.get(sourceInterval.a);
            int lineNum = firstToken.getLine();
            RawLineNum.add(Integer.toString(lineNum));
        }
        for (int i = 0; i < ctx.getChildCount(); i++) {
            ParseTree element = ctx.getChild(i);
            if (element instanceof RuleContext) {
                generateAST((RuleContext) element, verbose, indentation + (toBeIgnored ? 0 : 1), tokens);
            }
        }
    }

    private static void printDOT(PrintWriter writer) {
        printLabel(writer);
        int pos = 0;
        for (int i = 1; i < LineNum.size(); i++) {
            pos = getPos(Integer.parseInt(LineNum.get(i)) - 1, i);
            writer.println((Integer.parseInt(LineNum.get(i)) - 1) + Integer.toString(pos) + "->" + LineNum.get(i) + i);
        }
    }

    private static void printLabel(PrintWriter writer) {
        for (int i = 0; i < LineNum.size(); i++) {
//          writer.println(LineNum.get(i)+i+"[label=\""+Type.get(i)+"\\n "+Content.get(i)+" \"]");
            writer.println(LineNum.get(i) + i + "[label=\"" + Type.get(i) + "\", linenum=\"" + RawLineNum.get(i) + "\"]");
        }
    }

    private static int getPos(int n, int limit) {
        int pos = 0;
        for (int i = 0; i < limit; i++) {
            if (Integer.parseInt(LineNum.get(i)) == n) {
                pos = i;
            }
        }
        return pos;
    }
}

results is like this (ast_helloworld.dot):

digraph helloworld {
00[label="compilationUnit", linenum="1"]
11[label="normalClassDeclaration", linenum="1"]
22[label="classModifier", linenum="1"]
23[label="classBody", linenum="1"]
34[label="methodDeclaration", linenum="2"]
45[label="methodModifier", linenum="2"]
46[label="methodModifier", linenum="2"]
47[label="methodHeader", linenum="2"]
58[label="result", linenum="2"]
59[label="methodDeclarator", linenum="2"]
610[label="formalParameter", linenum="2"]
711[label="unannArrayType", linenum="2"]
812[label="unannClassType_lfno_unannClassOrInterfaceType", linenum="2"]
813[label="dims", linenum="2"]
714[label="variableDeclaratorId", linenum="2"]
415[label="block", linenum="2"]
516[label="expressionStatement", linenum="3"]
617[label="methodInvocation", linenum="3"]
718[label="typeName", linenum="3"]
819[label="packageOrTypeName", linenum="3"]
720[label="literal", linenum="3"]
00->11
11->22
11->23
23->34
34->45
34->46
34->47
47->58
47->59
59->610
610->711
711->812
711->813
610->714
34->415
415->516
516->617
617->718
718->819
617->720
}