Create mp4 files on Android using Jcodec

Question

i have some troubles with writing mp4 files on Android using MediaRecorder and Jcodec, here is my code

public class SequenceEncoder {
    private final static String CLASSTAG = SequenceEncoder.class.getSimpleName();

    private SeekableByteChannel ch;

    private byte[] yuv = null;

    private ArrayList<ByteBuffer> spsList;
    private ArrayList<ByteBuffer> ppsList;

    private CompressedTrack outTrack;

    private int frameNo;
    private MP4Muxer muxer;

    ArrayList<ByteBuffer> spsListTmp = new ArrayList<ByteBuffer>();
    ArrayList<ByteBuffer> ppsListTmp = new ArrayList<ByteBuffer>();

    // Encoder
    private MediaCodec mediaCodec = null;

    public SequenceEncoder(File out) throws IOException {
        this.ch = NIOUtils.writableFileChannel(out);

        // Muxer that will store the encoded frames
        muxer = new MP4Muxer(ch, Brand.MP4);

        // Add video track to muxer
        outTrack = muxer.addTrackForCompressed(TrackType.VIDEO, 25);

        // Encoder extra data ( SPS, PPS ) to be stored in a special place of
        // MP4
        spsList = new ArrayList<ByteBuffer>();
        ppsList = new ArrayList<ByteBuffer>();
    }

    @SuppressWarnings("unchecked")
    public void encodeImage(ByteBuffer buffer, int width, int height) throws IOException {
        if (yuv == null) {
            int bufferSize = width * height * 3 / 2;

            yuv = new byte[bufferSize];

            int bitRate = bufferSize;
            int frameRate = 25;
            String mimeType = "video/avc";

            // "video/avc"
            mediaCodec = MediaCodec.createEncoderByType(mimeType);
            MediaFormat mediaFormat = MediaFormat.createVideoFormat(mimeType, width, height);
            mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); // 125000);
            mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
            mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
            mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5);

            mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
            mediaCodec.start();
        }

        byte[] rgba = buffer.array();

        // Convert RGBA image to NV12 (YUV420SemiPlanar)
        Rgba2Yuv420.convert(rgba, yuv, width, height);

        synchronized (mediaCodec) {
        try {
            ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers();
            ByteBuffer[] outputBuffers = mediaCodec.getOutputBuffers();

            int inputBufferIndex = mediaCodec.dequeueInputBuffer(-1);
            if (inputBufferIndex >= 0) {
                ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
                inputBuffer.clear();
                inputBuffer.put(yuv);
                mediaCodec.queueInputBuffer(inputBufferIndex, 0,
                        yuv.length, 0, 0);
            }

            MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
            int outputBufferIndex = mediaCodec.dequeueOutputBuffer(
                    bufferInfo, 0);

            while (outputBufferIndex >= 0) {
                ByteBuffer outputBuffer = outputBuffers[outputBufferIndex];
                byte[] outData = new byte[bufferInfo.size];
                outputBuffer.get(outData);

                ByteBuffer frameBuffer = ByteBuffer.wrap(outData);

                spsListTmp.clear();
                ppsListTmp.clear();

                H264Utils.encodeMOVPacket(frameBuffer, spsListTmp, ppsListTmp);

                if (!spsListTmp.isEmpty())
                    spsList = (ArrayList<ByteBuffer>) spsListTmp.clone();
                if (!ppsListTmp.isEmpty())
                    ppsList = (ArrayList<ByteBuffer>) ppsListTmp.clone();

                outTrack.addFrame(new MP4Packet(frameBuffer, frameNo, 25, 1,
                        frameNo, true, null, frameNo, 0));

                frameNo++;

                mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
                outputBufferIndex = mediaCodec.dequeueOutputBuffer(
                        bufferInfo, 0);
            }

            if (outputBufferIndex < 0)
                switch (outputBufferIndex) {
                case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                    outputBuffers = mediaCodec.getOutputBuffers();
                    break;
                case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                    break;
                case MediaCodec.INFO_TRY_AGAIN_LATER:
                    break;
                default:
                    break;
                }
            } catch (Exception e) {
            }
        }
    }

    public void finish() throws IOException {
        if (!ch.isOpen())
            return;

        if (mediaCodec != null) {
            mediaCodec.stop();
            mediaCodec.release();
        }

        outTrack.addSampleEntry(H264Utils.createMOVSampleEntry(spsList, ppsList));

        // Write MP4 header and finalize recording
        muxer.writeHeader();
        NIOUtils.closeQuietly(ch);

        ch.close();
    }
}

As we can see Android MediaCodec expect YUV420SemiPlanar as input image, so i'm giving him the correct one. As a result i have a corrupted mp4 file with invalid colors, when i open this mp4 file from AVCon i see that color format in output file is yuv420p, so maybe that the problem? Please suggest how to fix this.

Also have another question, how to add compressed audio stream to muxer, have not found examples.

Answer 1

Android 4.3 (API 18) has two new features that may be useful.

First, the MediaCodec class accepts input from a Surface, so anything you can decode to a Surface or render with OpenGL ES can be recorded without having to fiddle with YUV color planes.

Second, the new MediaMuxer class provides a way to combine audio and H.264 video into a .mp4 file.

Sample source code (primarily for the video aspects) can be found here.

Answer 2

YUV420SemiPlanar of 4x4 image is a format like YYYYYYYYYYYYYYYYUVUVUVUV, Not YYYYYYYYYYYYYYYYUUUUVVVV. I can get mp4 file with proper color using Jcodec and MediaCodec on Android after I passed a image with the format.

I don't have an answer about audio.