`sasjs fs sync {localFolder} {remoteFolder}`

allanbowe commented 2 years ago

These are the steps to synchronise a remote SAS folder with a local one:

Run mp_hashdirectory() on the remote SAS server to get the file hashes
Hash the local directory
Compare the differences
Generate (sasjs fs compile) a program to deploy only the differences. This should also run mp_hashdirectory() again, for confirmation.
Run the program
Compare returned hashes and print any differences to the console

The SAS code to execute remote hashing is as follows (which assumes %let fsTarget=/some/target; is set at the start):

/* Get Hashes */
%mp_hashdirectory(&fsTarget,maxDepth=MAX,outds=work.hashes)

/* Prepare Response JSON */
filename tmp temp;
%mp_jsonout(OPEN,jref=tmp)
%mp_jsonout(OBJ,hashes,fmt=N,jref=tmp)
%mp_jsonout(CLOSE,jref=tmp)

/* Print to Log */
data _null_;
  retain eof;
  infile tmp end=eof lrecl=10000;
  if _n_=1 then putlog '>>weboutBEGIN<<';
  input;
  putlog _infile_;
  if eof then putlog '>>weboutEND<<';
run;

It requires the following macros to be compiled into the program:

An example of the returned JSON is below (embedded in the log). Note that the webout begin / end markers are at the START of the line (instances that are not at the start should be ignored)

>>weboutBEGIN<<
{"PROCESSED_DTTM" : "2022-10-13T14:09:07.264604"
, "hashes":
[
{"DIRECTORY":"/tmp/test2/services/common" ,"FILE_HASH":"D8CE7FC28229D708173F8C59B9D01011" ,"HASH_DURATION":0.0000698566 ,"FILE_PATH":"/tmp/test2/services/common/appinit.sas" ,"FILE_OR_FOLDER":"file" ,"LEVEL":2 }
,{"DIRECTORY":"/tmp/test2/services/common" ,"FILE_HASH":"ABB2509B193CE4C59A355099F8BC4EFB" ,"HASH_DURATION":0.0000700951 ,"FILE_PATH":"/tmp/test2/services/common/getdata.sas" ,"FILE_OR_FOLDER":"file" ,"LEVEL":2 }
,{"DIRECTORY":"/tmp/test2/services/files" ,"FILE_HASH":"FCDEA154C21BAC5B4478F2D270AC50B9" ,"HASH_DURATION":0.0000758171 ,"FILE_PATH":"/tmp/test2/services/files/upload.sas" ,"FILE_OR_FOLDER":"file" ,"LEVEL":2 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"2E471CF3A13DC41AE807B8A3711CC236" ,"HASH_DURATION":0.0000669956 ,"FILE_PATH":"/tmp/test2/doxy/Doxyfile" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"EA0878F5879E417BDA50978BE95D6D79" ,"HASH_DURATION":0.0001010895 ,"FILE_PATH":"/tmp/test2/doxy/DoxygenLayout.xml" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"FB9105ADD645369A65A88E86FCF31065" ,"HASH_DURATION":0.0000889301 ,"FILE_PATH":"/tmp/test2/doxy/doxygen.svg" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"C4E1FC2740412F664BE45EC6DF2D3463" ,"HASH_DURATION":0.0000698566 ,"FILE_PATH":"/tmp/test2/doxy/favicon.ico" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"4DFA27BA514DDA23480D582202A31CAD" ,"HASH_DURATION":0.0000770092 ,"FILE_PATH":"/tmp/test2/doxy/logo.png" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"44602E42A0841C6A26C5FC1484EB6066" ,"HASH_DURATION":0.0000710487 ,"FILE_PATH":"/tmp/test2/doxy/new_footer.html" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"7EBCBAC674034D260AA46528C5A0E812" ,"HASH_DURATION":0.0000739098 ,"FILE_PATH":"/tmp/test2/doxy/new_header.html" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/doxy" ,"FILE_HASH":"ADD967512054888AA978663223C7B6F8" ,"HASH_DURATION":0.0000731945 ,"FILE_PATH":"/tmp/test2/doxy/new_stylesheet.css" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/mocks" ,"FILE_HASH":"4EC249E0D30E52DFD3FED3FD72C2F885" ,"HASH_DURATION":0.0000660419 ,"FILE_PATH":"/tmp/test2/mocks/appinit.js" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/services" ,"FILE_HASH":"742419B33D15EE20C324959A7E20DC91" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/services/common" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/services" ,"FILE_HASH":"5FCB285AC821AC40A4DB0EB87B3B7A55" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/services/files" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2/services" ,"FILE_HASH":"5CFA46590D0F165177B70ADC282D647E" ,"HASH_DURATION":0.0002140999 ,"FILE_PATH":"/tmp/test2/services/serviceinit.sas" ,"FILE_OR_FOLDER":"file" ,"LEVEL":1 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"B14ABFE65CD47D2DF2E718167DF155FA" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":0 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"186187853B57E1C29D29E98711040E2A" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/doxy" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":0 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/macros" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":0 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"FF2AD3706F268742006FC10E911D82BF" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/mocks" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":0 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"119FEF4952299121014C4D694A5CA61A" ,"HASH_DURATION":0.0314488411 ,"FILE_PATH":"/tmp/test2/sasjsconfig.json" ,"FILE_OR_FOLDER":"file" ,"LEVEL":0 }
,{"DIRECTORY":"/tmp/test2" ,"FILE_HASH":"6B5E2B8B204D03357E6C1F05C2E3D60B" ,"HASH_DURATION":0 ,"FILE_PATH":"/tmp/test2/services" ,"FILE_OR_FOLDER":"folder" ,"LEVEL":0 }
]
}
>>weboutEND<<

allanbowe commented 2 years ago

Linked issue; https://github.com/sasjs/cli/issues/1262

sasjsbot commented 2 years ago

:tada: This issue has been resolved in version 3.20.0 :tada:

The release is available on:

Your semantic-release bot :package::rocket:

sasjs / cli

`sasjs fs sync {localFolder} {remoteFolder}` #1263