cubewise-code / bedrock

Bedrock is TM1 Best Practice assets built from years of TM1 experience
Other
79 stars 74 forks source link

}bedrock.cube.data.copy - Thread files created by main process obsolete (?) & sleep parameter wrong #361

Closed CWako closed 1 year ago

CWako commented 1 year ago

Describe the bug When using parallelisation, the main process will also create thread files for each thread, but these won't be deleted in the epilog, and are obsolete anyway, because only the sub-processes need to create thread files, which are monitored by the main process if they exist.

While( Scan( pEleDelim, sElementList ) > 0 );
    sSlicerEle = SubSt( sElementList, 1, Scan( pEleDelim, sElementList ) - 1 );
    sElementList = SubSt( sElementList, Scan( pEleDelim, sElementList ) + Long( pEleDelim ), Long( sElementList ) );
    # Do recursive process call with new RunProcess function
    nThreadElCounter = nThreadElCounter + 1;
    sDimDelim = If(pFilter @= '', '', pDimDelim );
    IF( nThreadElCounter = 1 );
      sFilter = Expand('%pFilter%%sDimDelim%%sDimParallel%%pEleStartDelim%%sSlicerEle%');
    ELSE;
      sFilter = Expand('%sFilter%%pEleDelim%%sSlicerEle%');
    ENDIF;
    IF( nThreadElCounter >= nElemsPerThread );
            nThreadID = INT( RAND( ) * 10000 + 1) + Numbr(cTimeStamp);
      sThreadControlFile = GetProcessName() | '_ThreadControlFile_' | cRandomInt | '_' | NumberToString(nThreadID) | '_' | cTimeStamp;
      AsciiOutput( cDir | sThreadControlFile | '.txt', '' );
      LogOutput( 'INFO', 'Executing subTI with Thread ID: ' | NumberToString(nThreadID) );
      RunProcess( cThisProcName, 'pLogoutput', pLogoutput,
        'pCube', pCube, 'pSrcView', pSrcView, 'pTgtView', pTgtView,
        'pFilter', sFilter, 'pFilterParallel', '', 'pEleMapping', pEleMapping, 'pMappingDelim', pMappingDelim,
        'pDimDelim', pDimDelim, 'pEleStartDelim', pEleStartDelim, 'pEleDelim', pEleDelim,
        'pFactor', pFactor, 'pSuppressConsol', pSuppressConsol, 'pSuppressConsolStrings', pSuppressConsolStrings, 'pSuppressRules', pSuppressRules, 'pSuppressZero', pSuppressZero, 'pCumulate', pCumulate,
        'pZeroTarget', pZeroTarget, 'pZeroSource', pZeroSource, 'pTemp', pTemp, 'pCubeLogging', pCubeLogging, 'pSandbox', pSandbox, 'pFile', pFile, 'pDecimalSeparator', pDecimalSeparator, 'pThousandSeparator', pThousandSeparator,
        'pThreadMode', 1, 'pThreadControlFile', sThreadControlFile
      );
      nThreadElCounter = 0;
      sFilter = '';
#**NOT NEEDED?**
      nThreadID = INT( RAND( ) * 10000 ) + 1;
          sThreadControlFile = GetProcessName() | '_ThreadControlFile_' | cRandomInt | '_' | NumberToString(nThreadID);
          AsciiOutput( cDir | sThreadControlFile | '.txt', '' );
          LogOutput( 'INFO', 'Executing subTI with Thread ID: ' | NumberToString(nThreadID) );
     ENDIF;
  End;

This monitoring is done in the epilog, but the parameter for the sleep command is only set to 1, which is one millisecond. It should be 1000 for one second. This causes the while loop to end much too early, and the main process to end before the sub-processes.

### Wait for all parallel threads to finish if using pFilterParallel
If( pFilterParallel @<> '' );
    sThreadFilePattern = GetProcessName() | '_ThreadControlFile_' | cRandomInt | '_' | '*.txt';
    LogOutput( 'INFO', 'Checking for: ' | sThreadFilePattern );
    i = 1;
    While( i < pMaxWaitSeconds );
        sThreadCheck = WildcardFileSearch( cDir | sThreadFilePattern, '' );
        If( sThreadCheck @<> '' );
            **Sleep( 1 );**
        Else;
            Break;
        EndIf;

    i = i + 1;
    End;
EndIf;