changeset 12441:1f44f1f1623d draft

Introduce CTU distortion refinement in analysis-save/load
author Aruna Matheswaran <aruna@multicorewareinc.com>
date Mon, 17 Dec 2018 16:49:08 +0530
parents 81373aab81df
children 56a73c852e25
files doc/reST/cli.rst source/CMakeLists.txt source/common/param.cpp source/encoder/analysis.cpp source/encoder/api.cpp source/encoder/encoder.cpp source/encoder/encoder.h source/x265.h source/x265cli.h
diffstat 9 files changed, 154 insertions(+-), 64 deletions(-) [+]
line wrap: on
line diff
--- a/doc/reST/cli.rst	Thu Dec 13 10:55:15 2018 +0530
+++ b/doc/reST/cli.rst	Mon Dec 17 16:49:08 2018 +0530
@@ -930,6 +930,14 @@ will not reuse analysis if slice type pa
 	Reuse MV information received through API call. Currently receives information for AVC size and the accepted 
 	string input is "avc". Default is disabled.
 
+.. option:: --refine-ctu-distortion <0/1>
+
+    Store/normalize ctu distortion in analysis-save/load.
+    0 - Disabled.
+    1 - Save ctu distortion to the analysis file specified during analysis-save.
+        Load CTU distortion from the analysis file and normalize it across every frame during analysis-load.
+    Default 0.
+
 .. option:: --scale-factor
 
 	Factor by which input video is scaled down for analysis save mode.
--- a/source/CMakeLists.txt	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/CMakeLists.txt	Mon Dec 17 16:49:08 2018 +0530
@@ -29,7 +29,7 @@ option(NATIVE_BUILD "Target the build CP
 option(STATIC_LINK_CRT "Statically link C runtime for release builds" OFF)
 mark_as_advanced(FPROFILE_USE FPROFILE_GENERATE NATIVE_BUILD)
 # X265_BUILD must be incremented each time the public API is changed
-set(X265_BUILD 168)
+set(X265_BUILD 169)
 configure_file("${PROJECT_SOURCE_DIR}/x265.def.in"
                "${PROJECT_BINARY_DIR}/x265.def")
 configure_file("${PROJECT_SOURCE_DIR}/x265_config.h.in"
--- a/source/common/param.cpp	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/common/param.cpp	Mon Dec 17 16:49:08 2018 +0530
@@ -297,6 +297,7 @@ void x265_param_default(x265_param* para
     param->interRefine = 0;
     param->bDynamicRefine = 0;
     param->mvRefine = 0;
+    param->ctuDistortionRefine = 0;
     param->bUseAnalysisFile = 1;
     param->csvfpt = NULL;
     param->forceFlush = 0;
@@ -1061,6 +1062,7 @@ int x265_param_parse(x265_param* p, cons
                 bError = true;
         }
         OPT("hrd-concat") p->bEnableHRDConcatFlag = atobool(value);
+        OPT("refine-ctu-distortion") p->ctuDistortionRefine = atoi(value);
         else
             return X265_PARAM_BAD_NAME;
     }
@@ -1416,6 +1418,8 @@ int x265_check_params(x265_param* param)
         "Invalid refine-inter value, refine-inter levels 0 to 3 supported");
     CHECK(param->intraRefine > 4 || param->intraRefine < 0,
         "Invalid refine-intra value, refine-intra levels 0 to 3 supported");
+    CHECK(param->ctuDistortionRefine < 0 || param->ctuDistortionRefine > 1,
+        "Invalid refine-ctu-distortion value, must be either 0 or 1");
     CHECK(param->maxAUSizeFactor < 0.5 || param->maxAUSizeFactor > 1.0,
         "Supported factor for controlling max AU size is from 0.5 to 1");
     CHECK((param->dolbyProfile != 0) && (param->dolbyProfile != 50),
@@ -1818,6 +1822,7 @@ char *x265_param2string(x265_param* p, i
     s += sprintf(s, " refine-intra=%d", p->intraRefine);
     s += sprintf(s, " refine-inter=%d", p->interRefine);
     s += sprintf(s, " refine-mv=%d", p->mvRefine);
+    s += sprintf(s, " refine-ctu-distortion=%d", p->ctuDistortionRefine);
     BOOL(p->bLimitSAO, "limit-sao");
     s += sprintf(s, " ctu-info=%d", p->bCTUInfo);
     BOOL(p->bLowPassDct, "lowpass-dct");
--- a/source/encoder/analysis.cpp	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/encoder/analysis.cpp	Mon Dec 17 16:49:08 2018 +0530
@@ -3560,7 +3560,7 @@ int Analysis::calculateQpforCuSize(const
     FrameData& curEncData = *m_frame->m_encData;
     double qp = baseQp >= 0 ? baseQp : curEncData.m_cuStat[ctu.m_cuAddr].baseQp;
 
-    if (m_param->analysisMultiPassDistortion && m_param->rc.bStatRead)
+    if ((m_param->analysisMultiPassDistortion && m_param->rc.bStatRead) || (m_param->ctuDistortionRefine && m_param->analysisLoad))
     {
         x265_analysis_distortion_data* distortionData = m_frame->m_analysisData.distortionData;
         if ((distortionData->threshold[ctu.m_cuAddr] < 0.9 || distortionData->threshold[ctu.m_cuAddr] > 1.1)
--- a/source/encoder/api.cpp	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/encoder/api.cpp	Mon Dec 17 16:49:08 2018 +0530
@@ -408,6 +408,7 @@ void x265_alloc_analysis_data(x265_param
     x265_analysis_inter_data *interData = analysis->interData = NULL;
     x265_analysis_intra_data *intraData = analysis->intraData = NULL;
     x265_analysis_distortion_data *distortionData = analysis->distortionData = NULL;
+
     bool isVbv = param->rc.vbvMaxBitrate > 0 && param->rc.vbvBufferSize > 0;
     int numDir = 2; //irrespective of P or B slices set direction as 2
     uint32_t numPlanes = param->internalCsp == X265_CSP_I400 ? 1 : 3;
@@ -419,18 +420,19 @@ void x265_alloc_analysis_data(x265_param
 #else
     uint32_t numCUs_sse_t = analysis->numCUsInFrame;
 #endif
-
-    //Allocate memory for distortionData pointer
-    CHECKED_MALLOC_ZERO(distortionData, x265_analysis_distortion_data, 1);
-    CHECKED_MALLOC_ZERO(distortionData->distortion, sse_t, analysis->numPartitions * numCUs_sse_t);
-    if (param->rc.bStatRead)
+    if (param->analysisMultiPassRefine || param->analysisMultiPassDistortion || param->ctuDistortionRefine)
     {
-        CHECKED_MALLOC_ZERO(distortionData->ctuDistortion, sse_t, numCUs_sse_t);
-        CHECKED_MALLOC_ZERO(distortionData->scaledDistortion, double, analysis->numCUsInFrame);
-        CHECKED_MALLOC_ZERO(distortionData->offset, double, analysis->numCUsInFrame);
-        CHECKED_MALLOC_ZERO(distortionData->threshold, double, analysis->numCUsInFrame);
+        //Allocate memory for distortionData pointer
+        CHECKED_MALLOC_ZERO(distortionData, x265_analysis_distortion_data, 1);
+        CHECKED_MALLOC_ZERO(distortionData->ctuDistortion, sse_t, analysis->numPartitions * numCUs_sse_t);
+        if (param->analysisLoad || param->rc.bStatRead)
+        {
+            CHECKED_MALLOC_ZERO(distortionData->scaledDistortion, double, analysis->numCUsInFrame);
+            CHECKED_MALLOC_ZERO(distortionData->offset, double, analysis->numCUsInFrame);
+            CHECKED_MALLOC_ZERO(distortionData->threshold, double, analysis->numCUsInFrame);
+        }
+        analysis->distortionData = distortionData;
     }
-    analysis->distortionData = distortionData;
 
     if (param->bDisableLookahead && isVbv)
     {
@@ -516,10 +518,9 @@ void x265_free_analysis_data(x265_param 
     //Free memory for distortionData pointers
     if (analysis->distortionData)
     {
-        X265_FREE((analysis->distortionData)->distortion);
-        if (param->rc.bStatRead)
+        X265_FREE((analysis->distortionData)->ctuDistortion);
+        if (param->rc.bStatRead || param->analysisLoad)
         {
-            X265_FREE((analysis->distortionData)->ctuDistortion);
             X265_FREE((analysis->distortionData)->scaledDistortion);
             X265_FREE((analysis->distortionData)->offset);
             X265_FREE((analysis->distortionData)->threshold);
--- a/source/encoder/encoder.cpp	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/encoder/encoder.cpp	Mon Dec 17 16:49:08 2018 +0530
@@ -1266,9 +1266,9 @@ int Encoder::encode(const x265_picture* 
                     pic_out->analysisData.wt = outFrame->m_analysisData.wt;
                     pic_out->analysisData.interData = outFrame->m_analysisData.interData;
                     pic_out->analysisData.intraData = outFrame->m_analysisData.intraData;
+                    pic_out->analysisData.distortionData = outFrame->m_analysisData.distortionData;
                     pic_out->analysisData.modeFlag[0] = outFrame->m_analysisData.modeFlag[0];
                     pic_out->analysisData.modeFlag[1] = outFrame->m_analysisData.modeFlag[1];
-                    pic_out->analysisData.distortionData = outFrame->m_analysisData.distortionData;
                     if (m_param->bDisableLookahead)
                     {
                         int factor = 1;
@@ -2839,6 +2839,20 @@ void Encoder::configure(x265_param *p)
         }
     }
 
+    if (p->ctuDistortionRefine == CTU_DISTORTION_INTERNAL)
+    {
+        if (!p->analysisLoad && !p->analysisSave)
+        {
+            x265_log(p, X265_LOG_WARNING, "refine-ctu-distortion 1 requires analysis save/load. Disabling refine-ctu-distortion\n");
+            p->ctuDistortionRefine = 0;
+        }
+        if (p->scaleFactor && p->analysisLoad)
+        {
+            x265_log(p, X265_LOG_WARNING, "refine-ctu-distortion 1 cannot be enabled along with multi resolution analysis refinement. Disabling refine-ctu-distortion\n");
+            p->ctuDistortionRefine = 0;
+        }
+    }
+
     if ((p->analysisMultiPassRefine || p->analysisMultiPassDistortion) && (p->bDistributeModeAnalysis || p->bDistributeMotionEstimation))
     {
         x265_log(p, X265_LOG_WARNING, "multi-pass-opt-analysis/multi-pass-opt-distortion incompatible with pmode/pme, Disabling pmode/pme\n");
@@ -3237,6 +3251,7 @@ void Encoder::readAnalysisFile(x265_anal
     const x265_analysis_data *picData = &(picIn->analysisData);
     x265_analysis_intra_data *intraPic = picData->intraData;
     x265_analysis_inter_data *interPic = picData->interData;
+    x265_analysis_distortion_data *picDistortion = picData->distortionData;
 
     int poc; uint32_t frameRecordSize;
     X265_FREAD(&frameRecordSize, sizeof(uint32_t), 1, m_analysisFileIn, &(picData->frameRecordSize));
@@ -3272,6 +3287,7 @@ void Encoder::readAnalysisFile(x265_anal
     X265_FREAD(&analysis->satdCost, sizeof(int64_t), 1, m_analysisFileIn, &(picData->satdCost));
     X265_FREAD(&analysis->numCUsInFrame, sizeof(int), 1, m_analysisFileIn, &(picData->numCUsInFrame));
     X265_FREAD(&analysis->numPartitions, sizeof(int), 1, m_analysisFileIn, &(picData->numPartitions));
+
     if (m_param->bDisableLookahead)
     {
         X265_FREAD(&analysis->numCuInHeight, sizeof(uint32_t), 1, m_analysisFileIn, &(picData->numCuInHeight));
@@ -3284,6 +3300,12 @@ void Encoder::readAnalysisFile(x265_anal
         analysis->numPartitions *= factor;
     /* Memory is allocated for inter and intra analysis data based on the slicetype */
     x265_alloc_analysis_data(m_param, analysis);
+
+    if (m_param->ctuDistortionRefine == CTU_DISTORTION_INTERNAL)
+    {
+        X265_FREAD((analysis->distortionData)->ctuDistortion, sizeof(sse_t), analysis->numCUsInFrame, m_analysisFileIn, picDistortion);
+        computeDistortionOffset(analysis);
+    }
     if (m_param->bDisableLookahead && m_rateControl->m_isVbv)
     {
         size_t vbvCount = m_param->lookaheadDepth + m_param->bframes + 2;
@@ -3532,6 +3554,7 @@ void Encoder::readAnalysisFile(x265_anal
     const x265_analysis_data *picData = &(picIn->analysisData);
     x265_analysis_intra_data *intraPic = picData->intraData;
     x265_analysis_inter_data *interPic = picData->interData;
+    x265_analysis_distortion_data *picDistortion = picData->distortionData;
 
     int poc; uint32_t frameRecordSize;
     X265_FREAD(&frameRecordSize, sizeof(uint32_t), 1, m_analysisFileIn, &(picData->frameRecordSize));
@@ -3567,6 +3590,7 @@ void Encoder::readAnalysisFile(x265_anal
     X265_FREAD(&analysis->satdCost, sizeof(int64_t), 1, m_analysisFileIn, &(picData->satdCost));
     X265_FREAD(&analysis->numCUsInFrame, sizeof(int), 1, m_analysisFileIn, &(picData->numCUsInFrame));
     X265_FREAD(&analysis->numPartitions, sizeof(int), 1, m_analysisFileIn, &(picData->numPartitions));
+    
     if (m_param->bDisableLookahead)
     {
         X265_FREAD(&analysis->numCuInHeight, sizeof(uint32_t), 1, m_analysisFileIn, &(picData->numCuInHeight));
@@ -3586,6 +3610,12 @@ void Encoder::readAnalysisFile(x265_anal
     /* Memory is allocated for inter and intra analysis data based on the slicetype */
     x265_alloc_analysis_data(m_param, analysis);
 
+    if (m_param->ctuDistortionRefine == CTU_DISTORTION_INTERNAL)
+    {
+        X265_FREAD((analysis->distortionData)->ctuDistortion, sizeof(sse_t), analysis->numCUsInFrame, m_analysisFileIn, picDistortion);
+        computeDistortionOffset(analysis);
+    }
+
     analysis->numPartitions = numPartitions * factor;
     analysis->numCUsInFrame = numCUsInFrame;
     analysis->numCuInHeight = numCuInHeight;
@@ -3956,6 +3986,7 @@ int Encoder::validateAnalysisData(x265_a
     X265_PARAM_VALIDATE(saveParam->chunkStart, sizeof(int), 1, &m_param->chunkStart, chunk-start);
     X265_PARAM_VALIDATE(saveParam->chunkEnd, sizeof(int), 1, &m_param->chunkEnd, chunk-end);
     X265_PARAM_VALIDATE(saveParam->cuTree,sizeof(int),1,&m_param->rc.cuTree, cutree - offset);
+    X265_PARAM_VALIDATE(saveParam->ctuDistortionRefine, sizeof(int), 1, &m_param->ctuDistortionRefine, ctu - distortion);
 
     int sourceHeight, sourceWidth;
     if (writeFlag)
@@ -4107,7 +4138,31 @@ int Encoder::getPuShape(puOrientation* p
     }
     return SIZE_2Nx2N;
 }
-
+void Encoder::computeDistortionOffset(x265_analysis_data* analysis)
+{
+    x265_analysis_distortion_data *distortionData = analysis->distortionData;
+
+    double sum = 0.0, sqrSum = 0.0;
+    for (uint32_t i = 0; i < analysis->numCUsInFrame; ++i)
+    {
+        distortionData->scaledDistortion[i] = X265_LOG2(X265_MAX(distortionData->ctuDistortion[i], 1));
+        sum += distortionData->scaledDistortion[i];
+        sqrSum += distortionData->scaledDistortion[i] * distortionData->scaledDistortion[i];
+    }
+    double avg = sum / analysis->numCUsInFrame;
+    distortionData->sdDistortion = pow(((sqrSum / analysis->numCUsInFrame) - (avg * avg)), 0.5);
+    distortionData->averageDistortion = avg;
+    distortionData->highDistortionCtuCount = distortionData->lowDistortionCtuCount = 0;
+    for (uint32_t i = 0; i < analysis->numCUsInFrame; ++i)
+    {
+        distortionData->threshold[i] = distortionData->scaledDistortion[i] / distortionData->averageDistortion;
+        distortionData->offset[i] = (distortionData->averageDistortion - distortionData->scaledDistortion[i]) / distortionData->sdDistortion;
+        if (distortionData->threshold[i] < 0.9 && distortionData->offset[i] >= 1)
+            distortionData->lowDistortionCtuCount++;
+        else if (distortionData->threshold[i] > 1.1 && distortionData->offset[i] <= -1)
+            distortionData->highDistortionCtuCount++;
+    }
+}
 void Encoder::readAnalysisFile(x265_analysis_data* analysis, int curPoc, int sliceType)
 {
 
@@ -4135,21 +4190,16 @@ void Encoder::readAnalysisFile(x265_anal
     /* Now arrived at the right frame, read the record */
     analysis->frameRecordSize = frameRecordSize;
     uint8_t* tempBuf = NULL, *depthBuf = NULL;
-    sse_t *tempdistBuf = NULL, *distortionBuf = NULL;
+    X265_FREAD((analysis->distortionData)->ctuDistortion, sizeof(sse_t), analysis->numCUsInFrame, m_analysisFileIn);
     tempBuf = X265_MALLOC(uint8_t, depthBytes);
     X265_FREAD(tempBuf, sizeof(uint8_t), depthBytes, m_analysisFileIn);
-    tempdistBuf = X265_MALLOC(sse_t, depthBytes);
-    X265_FREAD(tempdistBuf, sizeof(sse_t), depthBytes, m_analysisFileIn);
     depthBuf = tempBuf;
-    distortionBuf = tempdistBuf;
     x265_analysis_data *analysisData = (x265_analysis_data*)analysis;
     x265_analysis_intra_data *intraData = analysisData->intraData;
     x265_analysis_inter_data *interData = analysisData->interData;
-    x265_analysis_distortion_data *distortionData = analysisData->distortionData;
-
+
+    computeDistortionOffset(analysis);
     size_t count = 0;
-    uint32_t ctuCount = 0;
-    double sum = 0, sqrSum = 0;
     for (uint32_t d = 0; d < depthBytes; d++)
     {
         int bytes = analysis->numPartitions >> (depthBuf[d] * 2);
@@ -4157,30 +4207,10 @@ void Encoder::readAnalysisFile(x265_anal
             memset(&intraData->depth[count], depthBuf[d], bytes);
         else
             memset(&interData->depth[count], depthBuf[d], bytes);
-        distortionData->distortion[count] = distortionBuf[d];
-        distortionData->ctuDistortion[ctuCount] += distortionData->distortion[count];
         count += bytes;
-        if ((count % (unsigned)analysis->numPartitions) == 0)
-        {
-            distortionData->scaledDistortion[ctuCount] = X265_LOG2(X265_MAX(distortionData->ctuDistortion[ctuCount], 1));
-            sum += distortionData->scaledDistortion[ctuCount];
-            sqrSum += distortionData->scaledDistortion[ctuCount] * distortionData->scaledDistortion[ctuCount];
-            ctuCount++;
-        }
     }
-    double avg = sum / analysis->numCUsInFrame;
-    distortionData->sdDistortion = pow(((sqrSum / analysis->numCUsInFrame) - (avg * avg)), 0.5);
-    distortionData->averageDistortion = avg;
-    distortionData->highDistortionCtuCount = distortionData->lowDistortionCtuCount = 0;
-    for (uint32_t i = 0; i < analysis->numCUsInFrame; ++i)
-    {
-        distortionData->threshold[i] = distortionData->scaledDistortion[i] / distortionData->averageDistortion;
-        distortionData->offset[i] = (distortionData->averageDistortion - distortionData->scaledDistortion[i]) / distortionData->sdDistortion;
-        if (distortionData->threshold[i] < 0.9 && distortionData->offset[i] >= 1)
-            distortionData->lowDistortionCtuCount++;
-        else if (distortionData->threshold[i] > 1.1 && distortionData->offset[i] <= -1)
-            distortionData->highDistortionCtuCount++;
-    }
+
+
     if (!IS_X265_TYPE_I(sliceType))
     {
         MV *tempMVBuf[2], *MVBuf[2];
@@ -4233,11 +4263,27 @@ void Encoder::readAnalysisFile(x265_anal
         X265_FREE(tempModeBuf);
     }
     X265_FREE(tempBuf);
-    X265_FREE(tempdistBuf);
 
 #undef X265_FREAD
 }
 
+void Encoder::copyDistortionData(x265_analysis_data* analysis, FrameData &curEncData)
+{
+    for (uint32_t cuAddr = 0; cuAddr < analysis->numCUsInFrame; cuAddr++)
+    {
+        uint8_t depth = 0;
+        CUData* ctu = curEncData.getPicCTU(cuAddr);
+        x265_analysis_distortion_data *distortionData = (x265_analysis_distortion_data *)analysis->distortionData;
+        distortionData->ctuDistortion[cuAddr] = 0;
+        for (uint32_t absPartIdx = 0; absPartIdx < ctu->m_numPartitions;)
+        {
+            depth = ctu->m_cuDepth[absPartIdx];
+            distortionData->ctuDistortion[cuAddr] += ctu->m_distortion[absPartIdx];
+            absPartIdx += ctu->m_numPartitions >> (depth * 2);
+        }
+    }
+}
+
 void Encoder::writeAnalysisFile(x265_analysis_data* analysis, FrameData &curEncData)
 {
 
@@ -4273,8 +4319,15 @@ void Encoder::writeAnalysisFile(x265_ana
         analysis->frameRecordSize += sizeof(WeightParam) * numPlanes * numDir;
     }
 
+    if (m_param->ctuDistortionRefine == CTU_DISTORTION_INTERNAL)
+    {
+        copyDistortionData(analysis, curEncData);
+        analysis->frameRecordSize += analysis->numCUsInFrame * sizeof(sse_t);
+    }
+
     if (m_param->analysisReuseLevel > 1)
     {
+
         if (analysis->sliceType == X265_TYPE_IDR || analysis->sliceType == X265_TYPE_I)
         {
             for (uint32_t cuAddr = 0; cuAddr < analysis->numCUsInFrame; cuAddr++)
@@ -4406,6 +4459,8 @@ void Encoder::writeAnalysisFile(x265_ana
     X265_FWRITE(&analysis->satdCost, sizeof(int64_t), 1, m_analysisFileOut);
     X265_FWRITE(&analysis->numCUsInFrame, sizeof(int), 1, m_analysisFileOut);
     X265_FWRITE(&analysis->numPartitions, sizeof(int), 1, m_analysisFileOut);
+    if (m_param->ctuDistortionRefine == CTU_DISTORTION_INTERNAL)
+        X265_FWRITE((analysis->distortionData)->ctuDistortion, sizeof(sse_t), analysis->numCUsInFrame, m_analysisFileOut);
     if (analysis->sliceType > X265_TYPE_I)
         X265_FWRITE((WeightParam*)analysis->wt, sizeof(WeightParam), numPlanes * numDir, m_analysisFileOut);
 
@@ -4469,25 +4524,24 @@ void Encoder::writeAnalysisFileRefine(x2
     x265_analysis_inter_data *interData = analysisData->interData;
     x265_analysis_distortion_data *distortionData = analysisData->distortionData;
 
-    for (uint32_t cuAddr = 0; cuAddr < analysis->numCUsInFrame; cuAddr++)
+    copyDistortionData(analysis, curEncData);
+
+    if (curEncData.m_slice->m_sliceType == I_SLICE)
     {
-        uint8_t depth = 0;
-
-        CUData* ctu = curEncData.getPicCTU(cuAddr);
-
-        for (uint32_t absPartIdx = 0; absPartIdx < ctu->m_numPartitions; depthBytes++)
+        for (uint32_t cuAddr = 0; cuAddr < analysis->numCUsInFrame; cuAddr++)
         {
-            depth = ctu->m_cuDepth[absPartIdx];
-            if (curEncData.m_slice->m_sliceType == I_SLICE)
+            uint8_t depth = 0;
+            CUData* ctu = curEncData.getPicCTU(cuAddr);
+            for (uint32_t absPartIdx = 0; absPartIdx < ctu->m_numPartitions; depthBytes++)
+            {
+                depth = ctu->m_cuDepth[absPartIdx];
                 intraData->depth[depthBytes] = depth;
-            else
-                interData->depth[depthBytes] = depth;
-            distortionData->distortion[depthBytes] = ctu->m_distortion[absPartIdx];
-            absPartIdx += ctu->m_numPartitions >> (depth * 2);
+                absPartIdx += ctu->m_numPartitions >> (depth * 2);
+            }
         }
     }
 
-    if (curEncData.m_slice->m_sliceType != I_SLICE)
+    else
     {
         int32_t* ref[2];
         ref[0] = (analysis->interData)->ref;
@@ -4502,6 +4556,7 @@ void Encoder::writeAnalysisFileRefine(x2
             for (uint32_t absPartIdx = 0; absPartIdx < ctu->m_numPartitions; depthBytes++)
             {
                 depth = ctu->m_cuDepth[absPartIdx];
+                interData->depth[depthBytes] = depth;
                 interData->mv[0][depthBytes].word = ctu->m_mv[0][absPartIdx].word;
                 interData->mvpIdx[0][depthBytes] = ctu->m_mvpIdx[0][absPartIdx];
                 ref[0][depthBytes] = ctu->m_refIdx[0][absPartIdx];
@@ -4523,7 +4578,7 @@ void Encoder::writeAnalysisFileRefine(x2
     /* calculate frameRecordSize */
     analysis->frameRecordSize = sizeof(analysis->frameRecordSize) + sizeof(depthBytes) + sizeof(analysis->poc);
     analysis->frameRecordSize += depthBytes * sizeof(uint8_t);
-    analysis->frameRecordSize += depthBytes * sizeof(sse_t);
+    analysis->frameRecordSize += analysis->numCUsInFrame * sizeof(sse_t);
     if (curEncData.m_slice->m_sliceType != I_SLICE)
     {
         int numDir = (curEncData.m_slice->m_sliceType == P_SLICE) ? 1 : 2;
@@ -4535,6 +4590,7 @@ void Encoder::writeAnalysisFileRefine(x2
     X265_FWRITE(&analysis->frameRecordSize, sizeof(uint32_t), 1, m_analysisFileOut);
     X265_FWRITE(&depthBytes, sizeof(uint32_t), 1, m_analysisFileOut);
     X265_FWRITE(&analysis->poc, sizeof(uint32_t), 1, m_analysisFileOut);
+    X265_FWRITE(distortionData->ctuDistortion, sizeof(sse_t), analysis->numCUsInFrame, m_analysisFileOut);
     if (curEncData.m_slice->m_sliceType == I_SLICE)
     {
         X265_FWRITE((analysis->intraData)->depth, sizeof(uint8_t), depthBytes, m_analysisFileOut);
@@ -4543,7 +4599,6 @@ void Encoder::writeAnalysisFileRefine(x2
     {
         X265_FWRITE((analysis->interData)->depth, sizeof(uint8_t), depthBytes, m_analysisFileOut);
     }
-    X265_FWRITE(distortionData->distortion, sizeof(sse_t), depthBytes, m_analysisFileOut);
     if (curEncData.m_slice->m_sliceType != I_SLICE)
     {
         int numDir = curEncData.m_slice->m_sliceType == P_SLICE ? 1 : 2;
--- a/source/encoder/encoder.h	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/encoder/encoder.h	Mon Dec 17 16:49:08 2018 +0530
@@ -194,6 +194,8 @@ public:
     bool               m_reconfigure;      // Encoder reconfigure in progress
     bool               m_reconfigureRc;
 
+    int               m_saveCtuDistortionLevel;
+
     /* Begin intra refresh when one not in progress or else begin one as soon as the current 
      * one is done. Requires bIntraRefresh to be set.*/
     int                m_bQueuedIntraRefresh;
@@ -281,6 +283,8 @@ public:
 
     void readAnalysisFile(x265_analysis_data* analysis, int poc, const x265_picture* picIn, int paramBytes, cuLocation cuLoc);
 
+    void computeDistortionOffset(x265_analysis_data* analysis);
+
     int getCUIndex(cuLocation* cuLoc, uint32_t* count, int bytes, int flag);
 
     int getPuShape(puOrientation* puOrient, int partSize, int numCTU);
@@ -289,6 +293,8 @@ public:
 
     void writeAnalysisFileRefine(x265_analysis_data* analysis, FrameData &curEncData);
 
+    void copyDistortionData(x265_analysis_data* analysis, FrameData &curEncData);
+
     void finishFrameStats(Frame* pic, FrameEncoder *curEncoder, x265_frame_stats* frameStats, int inPoc);
 
     int validateAnalysisData(x265_analysis_data* analysis, int readWriteFlag);
--- a/source/x265.h	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/x265.h	Mon Dec 17 16:49:08 2018 +0530
@@ -131,6 +131,7 @@ typedef struct x265_analysis_validate
     int     chunkStart;
     int     chunkEnd;
     int     cuTree;
+    int     ctuDistortionRefine;
 }x265_analysis_validate;
 
 /* Stores intra analysis data for a single frame. This struct needs better packing */
@@ -182,9 +183,12 @@ typedef uint32_t sse_t;
 typedef uint64_t sse_t;
 #endif
 
+#define CTU_DISTORTION_OFF 0
+#define CTU_DISTORTION_INTERNAL 1
+#define CTU_DISTORTION_EXTERNAL 2
+
 typedef struct x265_analysis_distortion_data
 {
-    sse_t*        distortion;
     sse_t*        ctuDistortion;
     double*       scaledDistortion;
     double        averageDistortion;
@@ -193,6 +197,7 @@ typedef struct x265_analysis_distortion_
     uint32_t      lowDistortionCtuCount;
     double*       offset;
     double*       threshold;
+
 }x265_analysis_distortion_data;
 
 /* Stores all analysis data for a single frame */
@@ -1736,6 +1741,11 @@ typedef struct x265_param
     /* Set concantenation flag for the first keyframe in the HRD buffering period SEI. */
     int bEnableHRDConcatFlag;
 
+
+    /* Store/normalize ctu distortion in analysis-save/load. Ranges from 0 - 1.
+    *  0 - Disabled. 1 - Save/Load ctu distortion to/from the file specified 
+    * analysis-save/load. Default 0. */
+    int       ctuDistortionRefine;
 } x265_param;
 /* x265_param_alloc:
  *  Allocates an x265_param instance. The returned param structure is not
--- a/source/x265cli.h	Thu Dec 13 10:55:15 2018 +0530
+++ b/source/x265cli.h	Mon Dec 17 16:49:08 2018 +0530
@@ -291,6 +291,7 @@ static const struct option long_options[
     { "dolby-vision-profile",  required_argument, NULL, 0 },
     { "refine-mv",            no_argument, NULL, 0 },
     { "no-refine-mv",         no_argument, NULL, 0 },
+    { "refine-ctu-distortion", required_argument, NULL, 0 },
     { "force-flush",    required_argument, NULL, 0 },
     { "splitrd-skip",         no_argument, NULL, 0 },
     { "no-splitrd-skip",      no_argument, NULL, 0 },
@@ -514,6 +515,10 @@ static void showHelp(x265_param *param)
         "                                Default:%d\n", param->interRefine);
     H0("   --[no-]dynamic-refine         Dynamically changes refine-inter level for each CU. Default %s\n", OPT(param->bDynamicRefine));
     H0("   --[no-]refine-mv              Enable mv refinement for load mode. Default %s\n", OPT(param->mvRefine));
+    H0("   --refine-ctu-distortion       Store/normalize ctu distortion in analysis-save/load.\n"
+        "                                    - 0 : Disabled.\n"
+        "                                    - 1 : Store/Load ctu distortion to/from the file specified in analysis-save/load.\n"
+        "                                Default 0 - Disabled\n");
     H0("   --aq-mode <integer>           Mode for Adaptive Quantization - 0:none 1:uniform AQ 2:auto variance 3:auto variance with bias to dark scenes. Default %d\n", param->rc.aqMode);
     H0("   --aq-strength <float>         Reduces blocking and blurring in flat and textured areas (0 to 3.0). Default %.2f\n", param->rc.aqStrength);
     H0("   --[no-]aq-motion              Adaptive Quantization based on the relative motion of each CU w.r.t., frame. Default %s\n", OPT(param->bOptCUDeltaQP));