changeset 12365:182914e1d201 draft

Add support for chunked encoding
author Aruna Matheswaran <aruna@multicorewareinc.com>
date Wed, 14 Mar 2018 12:30:28 +0530
parents 03dcb3457b7e
children 3364f438c702
files doc/reST/cli.rst source/CMakeLists.txt source/common/param.cpp source/encoder/api.cpp source/encoder/encoder.cpp source/encoder/slicetype.cpp source/test/regression-tests.txt source/x265.cpp source/x265.h source/x265cli.h
diffstat 10 files changed, 89 insertions(+-), 15 deletions(-) [+]
line wrap: on
line diff
--- a/doc/reST/cli.rst	Mon May 28 14:05:25 2018 +0530
+++ b/doc/reST/cli.rst	Wed Mar 14 12:30:28 2018 +0530
@@ -535,6 +535,20 @@ frame counts) are only applicable to the
 
 	**CLI ONLY**
 
+.. option:: --chunk-start <integer>
+
+	First frame of the chunk. Frames preceeding this in display order will
+	be encoded, however, they will be discarded in the bitstream. This
+	feature can be enabled only in closed GOP structures.
+	Default 0 (disabled).
+	
+.. option:: --chunk-end <integer>
+
+	Last frame of the chunk. Frames following this in display order will be
+	used in taking lookahead decisions, but, they will not be encoded.
+	This feature can be enabled only in closed GOP structures.
+	Default 0 (disabled).
+
 Profile, Level, Tier
 ====================
 
--- a/source/CMakeLists.txt	Mon May 28 14:05:25 2018 +0530
+++ b/source/CMakeLists.txt	Wed Mar 14 12:30:28 2018 +0530
@@ -29,7 +29,7 @@ option(NATIVE_BUILD "Target the build CP
 option(STATIC_LINK_CRT "Statically link C runtime for release builds" OFF)
 mark_as_advanced(FPROFILE_USE FPROFILE_GENERATE NATIVE_BUILD)
 # X265_BUILD must be incremented each time the public API is changed
-set(X265_BUILD 160)
+set(X265_BUILD 161)
 configure_file("${PROJECT_SOURCE_DIR}/x265.def.in"
                "${PROJECT_BINARY_DIR}/x265.def")
 configure_file("${PROJECT_SOURCE_DIR}/x265_config.h.in"
--- a/source/common/param.cpp	Mon May 28 14:05:25 2018 +0530
+++ b/source/common/param.cpp	Wed Mar 14 12:30:28 2018 +0530
@@ -156,6 +156,9 @@ void x265_param_default(x265_param* para
     param->lookaheadThreads = 0;
     param->scenecutBias = 5.0;
     param->radl = 0;
+    param->chunkStart = 0;
+    param->chunkEnd = 0;
+
     /* Intra Coding Tools */
     param->bEnableConstrainedIntra = 0;
     param->bEnableStrongIntraSmoothing = 1;
@@ -1043,6 +1046,8 @@ int x265_param_parse(x265_param* p, cons
         OPT("single-sei") p->bSingleSeiNal = atobool(value);
 		OPT("atc-sei") p->preferredTransferCharacteristics = atoi(value);
 		OPT("pic-struct") p->pictureStructure = atoi(value);
+        OPT("chunk-start") p->chunkStart = atoi(value);
+        OPT("chunk-end") p->chunkEnd = atoi(value);
         else
             return X265_PARAM_BAD_NAME;
     }
@@ -1603,6 +1608,10 @@ char *x265_param2string(x265_param* p, i
     s += sprintf(s, " input-res=%dx%d", p->sourceWidth - padx, p->sourceHeight - pady);
     s += sprintf(s, " interlace=%d", p->interlaceMode);
     s += sprintf(s, " total-frames=%d", p->totalFrames);
+    if (p->chunkStart)
+        s += sprintf(s, " chunk-start=%d", p->chunkStart);
+    if (p->chunkEnd)
+        s += sprintf(s, " chunk-end=%d", p->chunkEnd);
     s += sprintf(s, " level-idc=%d", p->levelIdc);
     s += sprintf(s, " high-tier=%d", p->bHighTier);
     s += sprintf(s, " uhd-bd=%d", p->uhdBluray);
--- a/source/encoder/api.cpp	Mon May 28 14:05:25 2018 +0530
+++ b/source/encoder/api.cpp	Wed Mar 14 12:30:28 2018 +0530
@@ -281,7 +281,7 @@ int x265_encoder_encode(x265_encoder *en
         pic_in->analysis2Pass.analysisFramedata = NULL;
     }
 
-    if (pp_nal && numEncoded > 0)
+    if (pp_nal && numEncoded > 0 && encoder->m_outputCount >= encoder->m_latestParam->chunkStart)
     {
         *pp_nal = &encoder->m_nalList.m_nal[0];
         if (pi_nal) *pi_nal = encoder->m_nalList.m_numNal;
@@ -289,7 +289,7 @@ int x265_encoder_encode(x265_encoder *en
     else if (pi_nal)
         *pi_nal = 0;
 
-    if (numEncoded && encoder->m_param->csvLogLevel)
+    if (numEncoded && encoder->m_param->csvLogLevel && encoder->m_outputCount >= encoder->m_latestParam->chunkStart)
         x265_csvlog_frame(encoder->m_param, pic_out);
 
     if (numEncoded < 0)
--- a/source/encoder/encoder.cpp	Mon May 28 14:05:25 2018 +0530
+++ b/source/encoder/encoder.cpp	Wed Mar 14 12:30:28 2018 +0530
@@ -889,7 +889,7 @@ int Encoder::encode(const x265_picture* 
         m_exportedPic = NULL;
         m_dpb->recycleUnreferenced();
     }
-    if (pic_in)
+    if (pic_in && (!m_param->chunkEnd || (m_encodedFrameNum < m_param->chunkEnd)))
     {
         if (m_latestParam->forceFlush == 1)
         {
@@ -1319,7 +1319,8 @@ int Encoder::encode(const x265_picture* 
             if (m_aborted)
                 return -1;
 
-            finishFrameStats(outFrame, curEncoder, frameData, m_pocLast);
+            if ((m_outputCount + 1)  >= m_param->chunkStart)
+                finishFrameStats(outFrame, curEncoder, frameData, m_pocLast);
 
             /* Write RateControl Frame level stats in multipass encodes */
             if (m_param->rc.bStatWrite)
@@ -1354,8 +1355,12 @@ int Encoder::encode(const x265_picture* 
             }
             else
                 m_exportedPic = outFrame;
-
-            m_numDelayedPic--;
+            
+            m_outputCount++;
+            if (m_param->chunkEnd == m_outputCount)
+                m_numDelayedPic = 0;
+            else 
+                m_numDelayedPic--;
 
             ret = 1;
         }
@@ -1364,7 +1369,7 @@ int Encoder::encode(const x265_picture* 
          * curEncoder is guaranteed to be idle at this point */
         if (!pass)
             frameEnc = m_lookahead->getDecidedPicture();
-        if (frameEnc && !pass)
+        if (frameEnc && !pass && (!m_param->chunkEnd || (m_encodedFrameNum < m_param->chunkEnd)))
         {
             if (m_param->analysisMultiPassRefine || m_param->analysisMultiPassDistortion)
             {
@@ -1485,6 +1490,7 @@ int Encoder::encode(const x265_picture* 
             frameEnc->m_encData->m_slice->m_iNumRPSInSPS = m_sps.spsrpsNum;
 
             curEncoder->m_rce.encodeOrder = frameEnc->m_encodeOrder = m_encodedFrameNum++;
+
             if (!m_param->analysisLoad || !m_param->bDisableLookahead)
             {
                 if (m_bframeDelay)
@@ -2123,7 +2129,7 @@ void Encoder::finishFrameStats(Frame* cu
     {
         const int picOrderCntLSB = slice->m_poc - slice->m_lastIDR;
 
-        frameStats->encoderOrder = m_outputCount++;
+        frameStats->encoderOrder = m_outputCount;
         frameStats->sliceType = c;
         frameStats->poc = picOrderCntLSB;
         frameStats->qp = curEncData.m_avgQpAq;
@@ -3124,6 +3130,19 @@ void Encoder::configure(x265_param *p)
         p->radl = 0;
         x265_log(p, X265_LOG_WARNING, "Radl requires fixed gop-length (keyint == min-keyint). Disabling radl.\n");
     }
+
+    if ((p->chunkStart || p->chunkEnd) && p->bOpenGOP)
+    {
+        p->chunkStart = p->chunkEnd = 0;
+        x265_log(p, X265_LOG_WARNING, "Chunking requires closed gop structure. Disabling chunking.\n");
+    }
+
+    if (p->chunkEnd < p->chunkStart)
+    {
+        p->chunkStart = p->chunkEnd = 0;
+        x265_log(p, X265_LOG_WARNING, "chunk-end cannot be less than chunk-start. Disabling chunking.\n");
+    }
+
 }
 
 void Encoder::allocAnalysis(x265_analysis_data* analysis)
--- a/source/encoder/slicetype.cpp	Mon May 28 14:05:25 2018 +0530
+++ b/source/encoder/slicetype.cpp	Wed Mar 14 12:30:28 2018 +0530
@@ -1108,8 +1108,9 @@ void Lookahead::slicetypeDecide()
             x265_log(m_param, X265_LOG_WARNING, "B-ref at frame %d incompatible with B-pyramid and %d reference frames\n",
                      frm.sliceType, m_param->maxNumReferences);
         }
-        if ((!m_param->bIntraRefresh || frm.frameNum == 0) && frm.frameNum - m_lastKeyframe >= m_param->keyframeMax &&
-            (!m_extendGopBoundary || frm.frameNum - m_lastKeyframe >= m_param->keyframeMax + m_param->gopLookahead))
+        if (((!m_param->bIntraRefresh || frm.frameNum == 0) && frm.frameNum - m_lastKeyframe >= m_param->keyframeMax &&
+            (!m_extendGopBoundary || frm.frameNum - m_lastKeyframe >= m_param->keyframeMax + m_param->gopLookahead)) ||
+            (frm.frameNum == (m_param->chunkStart - 1)) || (frm.frameNum == m_param->chunkEnd))
         {
             if (frm.sliceType == X265_TYPE_AUTO || frm.sliceType == X265_TYPE_I)
                 frm.sliceType = m_param->bOpenGOP && m_lastKeyframe >= 0 ? X265_TYPE_I : X265_TYPE_IDR;
@@ -1123,7 +1124,7 @@ void Lookahead::slicetypeDecide()
                 frm.sliceType = m_param->bOpenGOP && m_lastKeyframe >= 0 ? X265_TYPE_I : X265_TYPE_IDR;
             }
         }
-        if (frm.sliceType == X265_TYPE_I && frm.frameNum - m_lastKeyframe >= m_param->keyframeMin)
+        if ((frm.sliceType == X265_TYPE_I && frm.frameNum - m_lastKeyframe >= m_param->keyframeMin) || (frm.frameNum == (m_param->chunkStart - 1)) || (frm.frameNum == m_param->chunkEnd))
         {
             if (m_param->bOpenGOP)
             {
@@ -1416,7 +1417,19 @@ void Lookahead::slicetypeAnalyse(Lowres 
         return;
     }
     frames[framecnt + 1] = NULL;
-    int keyFrameLimit = m_param->keyframeMax + m_lastKeyframe - frames[0]->frameNum - 1;
+
+    int keylimit = m_param->keyframeMax;
+    if (frames[0]->frameNum < m_param->chunkEnd)
+    {
+        int chunkStart = (m_param->chunkStart - m_lastKeyframe - 1);
+        int chunkEnd = (m_param->chunkEnd - m_lastKeyframe);
+        if ((chunkStart > 0) && (chunkStart < m_param->keyframeMax))
+            keylimit = chunkStart;
+        else if ((chunkEnd > 0) && (chunkEnd < m_param->keyframeMax))
+            keylimit = chunkEnd;
+    }
+
+    int keyFrameLimit = keylimit + m_lastKeyframe - frames[0]->frameNum - 1;
     if (m_param->gopLookahead && keyFrameLimit <= m_param->bframes + 1)
         keyintLimit = keyFrameLimit + m_param->gopLookahead;
     else
--- a/source/test/regression-tests.txt	Mon May 28 14:05:25 2018 +0530
+++ b/source/test/regression-tests.txt	Wed Mar 14 12:30:28 2018 +0530
@@ -173,4 +173,7 @@ crowd_run_1080p50.y4m,--preset medium --
 RaceHorses_416x240_30.y4m,--preset slow --no-cutree --ctu 16 --analysis-save x265_analysis.dat --analysis-reuse-level 10 --scale-factor 2 --crf 22  --vbv-maxrate 1000 --vbv-bufsize 1000::RaceHorses_832x480_30.y4m, --preset slow --no-cutree --ctu 32 --analysis-load x265_analysis.dat  --analysis-save x265_analysis_2.dat --analysis-reuse-level 10 --scale-factor 2 --crf 16 --vbv-maxrate 4000 --vbv-bufsize 4000 --refine-intra 0 --refine-inter 1::RaceHorses_1664x960_30.y4m,--preset slow --no-cutree --ctu 64 --analysis-load x265_analysis_2.dat  --analysis-reuse-level 10 --scale-factor 2 --crf 12 --vbv-maxrate 7000 --vbv-bufsize 7000 --refine-intra 2 --refine-inter 2
 ElFunete_960x540_60.yuv,--colorprim bt709 --transfer bt709 --chromaloc 2 --aud --repeat-headers --no-opt-qp-pps --no-opt-ref-list-length-pps --wpp --no-interlace --sar 1:1 --min-keyint 60 --no-open-gop --rc-lookahead 180 --bframes 5 --b-intra --ref 4 --cbqpoffs -2 --crqpoffs -2 --lookahead-threads 0 --weightb --qg-size 8 --me star --preset veryslow --frame-threads 1 --b-adapt 2 --aq-mode 3 --rd 6 --pools 15 --colormatrix bt709 --keyint 120 --high-tier --ctu 64 --tune psnr --bitrate 10000 --vbv-bufsize 30000 --vbv-maxrate 17500 --analysis-reuse-level 10 --analysis-save elfuente_960x540.dat --scale-factor 2::ElFunete_1920x1080_60.yuv,--colorprim bt709 --transfer bt709 --chromaloc 2 --aud --repeat-headers --no-opt-qp-pps --no-opt-ref-list-length-pps --wpp --no-interlace --sar 1:1 --min-keyint 60 --no-open-gop --rc-lookahead 180 --bframes 5 --b-intra --ref 4 --cbqpoffs -2 --crqpoffs -2 --lookahead-threads 0 --weightb --qg-size 8 --me star --preset veryslow --frame-threads 1 --b-adapt 2 --aq-mode 3 --rd 6 --pools 15 --colormatrix bt709 --keyint 120 --high-tier --ctu 64 --tune psnr --bitrate 10000 --vbv-bufsize 30000 --vbv-maxrate 17500 --analysis-reuse-level 10 --analysis-save elfuente_1920x1080.dat --limit-tu 0 --scale-factor 2 --analysis-load elfuente_960x540.dat --refine-intra 4 --refine-inter 2::ElFuente_3840x2160_60.yuv,--colorprim bt709 --transfer bt709 --chromaloc 2 --aud --repeat-headers --no-opt-qp-pps --no-opt-ref-list-length-pps --wpp --no-interlace --sar 1:1 --min-keyint 60 --no-open-gop --rc-lookahead 180 --bframes 5 --b-intra --ref 4 --cbqpoffs -2 --crqpoffs -2 --lookahead-threads 0 --weightb --qg-size 8 --me star --preset veryslow --frame-threads 1 --b-adapt 2 --aq-mode 3 --rd 6 --pools 15 --colormatrix bt709 --keyint 120 --high-tier --ctu 64 --tune=psnr --bitrate 24000 --vbv-bufsize 84000 --vbv-maxrate 49000 --analysis-reuse-level 10 --limit-tu 0 --scale-factor 2 --analysis-load elfuente_1920x1080.dat --refine-intra 4 --refine-inter 2
 
+#segment encoding
+BasketballDrive_1920x1080_50.y4m, --preset ultrafast --no-open-gop --chunk-start 100 --chunk-end 200
+
 # vim: tw=200
--- a/source/x265.cpp	Mon May 28 14:05:25 2018 +0530
+++ b/source/x265.cpp	Wed Mar 14 12:30:28 2018 +0530
@@ -144,7 +144,7 @@ void CLIOptions::printStatus(uint32_t fr
     {
         int eta = (int)(elapsed * (framesToBeEncoded - frameNum) / ((int64_t)frameNum * 1000000));
         sprintf(buf, "x265 [%.1f%%] %d/%d frames, %.2f fps, %.2f kb/s, eta %d:%02d:%02d",
-                100. * frameNum / framesToBeEncoded, frameNum, framesToBeEncoded, fps, bitrate,
+            100. * frameNum / (param->chunkEnd ? param->chunkEnd : param->totalFrames), frameNum, (param->chunkEnd ? param->chunkEnd : param->totalFrames), fps, bitrate,
                 eta / 3600, (eta / 60) % 60, eta % 60);
     }
     else
@@ -403,7 +403,7 @@ bool CLIOptions::parse(int argc, char **
     if (this->framesToBeEncoded == 0 && info.frameCount > (int)seek)
         this->framesToBeEncoded = info.frameCount - seek;
     param->totalFrames = this->framesToBeEncoded;
-
+    
     /* Force CFR until we have support for VFR */
     info.timebaseNum = param->fpsDenom;
     info.timebaseDenom = param->fpsNum;
--- a/source/x265.h	Mon May 28 14:05:25 2018 +0530
+++ b/source/x265.h	Wed Mar 14 12:30:28 2018 +0530
@@ -1638,6 +1638,18 @@ typedef struct x265_param
 
     /* Enable writing all SEI messgaes in one single NAL instead of mul*/
     int       bSingleSeiNal;
+
+
+    /* First frame of the chunk. Frames preceeding this in display order will
+	* be encoded, however, they will be discarded in the bitstream.
+    * Default 0 (disabled). */
+    int       chunkStart;
+
+    /* Last frame of the chunk. Frames following this in display order will be
+	* used in taking lookahead decisions, but, they will not be encoded.
+    * Default 0 (disabled). */
+    int       chunkEnd;
+
 } x265_param;
 
 /* x265_param_alloc:
--- a/source/x265cli.h	Mon May 28 14:05:25 2018 +0530
+++ b/source/x265cli.h	Wed Mar 14 12:30:28 2018 +0530
@@ -152,6 +152,8 @@ static const struct option long_options[
     { "vbv-init",       required_argument, NULL, 0 },
     { "vbv-end",        required_argument, NULL, 0 },
     { "vbv-end-fr-adj", required_argument, NULL, 0 },
+    { "chunk-start",    required_argument, NULL, 0 },
+    { "chunk-end",      required_argument, NULL, 0 },
     { "bitrate",        required_argument, NULL, 0 },
     { "qp",             required_argument, NULL, 'q' },
     { "aq-mode",        required_argument, NULL, 0 },
@@ -468,6 +470,8 @@ static void showHelp(x265_param *param)
     H0("   --vbv-init <float>            Initial VBV buffer occupancy (fraction of bufsize or in kbits). Default %.2f\n", param->rc.vbvBufferInit);
     H0("   --vbv-end <float>             Final VBV buffer emptiness (fraction of bufsize or in kbits). Default 0 (disabled)\n");
     H0("   --vbv-end-fr-adj <float>      Frame from which qp has to be adjusted to achieve final decode buffer emptiness. Default 0\n");
+    H0("   --chunk-start <integer>       First frame of the chunk. Default 0 (disabled)\n");
+    H0("   --chunk-end <integer>         Last frame of the chunk. Default 0 (disabled)\n");
     H0("   --pass                        Multi pass rate control.\n"
        "                                   - 1 : First pass, creates stats file\n"
        "                                   - 2 : Last pass, does not overwrite stats file\n"