medamineziraoui
commented
4 years ago HOW TO TEST WIP:
The release : 2.0.28
so first you need to use the projectsExportV1 to export job id that have many versions in the v1 version.
Then you need to use the projectsImport to import the job with this build.
import io.saagie.plugin.dataops.tasks.projects.artifact.ProjectsExportJobV1Task
import io.saagie.plugin.dataops.tasks.projects.artifact.ProjectsImportJobTask
import groovy.json.JsonSlurper;
plugins {
id 'groovy'
id 'base'
id 'io.saagie.gradle-saagie-dataops-plugin' version '2.0.28'
}
dependencies {
implementation group: 'gradle.plugin.io.saagie', name: 'gradle-saagie-dataops-plugin', version: '2.0.28'
}
def exportLocation = '/test_gradle/' // <=== edit this with the export path
def tmpLocation = './tmp' // <=== edit this with the tmp file path
task projectExportSqoop(type: ProjectsExportJobV1Task) { // <====== Create the first job
configuration = saagie {}
doFirst {
projectExportSqoop.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['22186']
}
exportArtifacts {
export_file = exportLocation + '/job/sqoop.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportSqoop'
group = 'Saagie'
description= 'export v1 sqoop job for saagie'
}
task projectExportSpark(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportSpark.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['4708']
include_all_versions=true
}
exportArtifacts {
export_file = exportLocation + '/job/spark.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportSpark'
group = 'Saagie'
description= 'export v1 spark job for saagie'
}
task projectExportJava(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportJava.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['12869']
}
exportArtifacts {
export_file = exportLocation + '/job/java.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportJava'
group = 'Saagie'
description= 'export v1 Java job for saagie'
}
task projectExportR(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportR.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['20983']
}
exportArtifacts {
export_file = exportLocation + '/job/r.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportR'
group = 'Saagie'
description= 'export v1 R job for saagie'
}
task projectExportTalend(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportTalend.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['21439']
}
exportArtifacts {
export_file = exportLocation + '/job/talend.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportTalend'
group = 'Saagie'
description= 'export v1 talend job for saagie'
}
task projectExportPython(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportPython.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['22702']
}
exportArtifacts {
export_file = exportLocation + '/job/python.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportPython'
group = 'Saagie'
description= 'export v1 python job for saagie'
}
task projectExportDocker(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportDocker.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
job {
ids = ['22712']
}
exportArtifacts {
export_file = exportLocation + '/job/docker.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportDocker'
group = 'Saagie'
description= 'export v1 Docker job for saagie'
}
task projectExportPipelineWithJobandVersion(type: ProjectsExportJobV1Task) { // <====== Create the second job
configuration = saagie {}
doFirst {
projectExportPipelineWithJobandVersion.configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
acceptSelfSigned = true
}
pipeline {
ids = ['484']
include_job=true
include_all_versions=true
}
exportArtifacts {
export_file = exportLocation + '/pipeline/pipelinejobver.zip'
overwrite=true
temporary_directory= tmpLocation
}
}
}
taskName = 'projectExportPipelineWithJobandVersion'
group = 'Saagie'
description= 'export Pipleline with job and version dependencies'
}
task projectImportSqoop( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportSqoop'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportSqoop.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/sqoop.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import sqoop job for saagie'
taskName = 'projectImportSqoop'
}
task projectImportSpark( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportSpark'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportSpark.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/spark.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import sqoop job for saagie'
taskName = 'projectImportSpark'
}
task projectImportJava( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportJava'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportJava.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/java.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import java job for saagie'
taskName = 'projectImportJava'
}
task projectImportR( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportR'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportR.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/r.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import R job for saagie'
taskName = 'projectImportR'
}
task projectImportTalend( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportTalend'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportTalend.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/talend.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import Talend job for saagie'
taskName = 'projectImportTalend'
}
task projectImportPython( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportPython'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportPython.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/python.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import Python job for saagie'
taskName = 'projectImportPython'
}
task projectImportDocker( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportDocker'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportDocker.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/job/docker.zip'
temporary_directory= tmpLocation
}
}
}
description= 'import Docker job for saagie'
taskName = 'projectImportDocker'
}
task projectImportPipelineWithJobandVersion( // <======== Import artifacts, see that we use the gradle dependOn
type: ProjectsImportJobTask) {
dependsOn(['projectExportPipelineWithJobandVersion'])
configuration = saagie {
server {
url = REPLACE_SAAGIEURL
login = REPLACE_SAAGIELOGIN
password = REPLACE_SAAGIEPASSWORD
environment = REPLACE_SAAGIE
jwt = true
}
}
doFirst {
projectImportPipelineWithJobandVersion.configuration = saagie {
project {
id = saagieprojectid
}
importArtifacts {
import_file = exportLocation + '/pipeline/pipelinejobver.zip'
temporary_directory= tmpLocation
}
}
}
description= 'Import Pipeline with Dependency Job and all versions'
taskName = 'projectImportPipelineWithJobandVersion'
}
task projectTestAll( // <======== Import artifacts, see that we use the gradle dependOn
type:Exec) {
dependsOn(['projectImportSqoop','projectImportSpark','projectImportJava','projectImportR','projectImportTalend','projectImportPython','projectImportDocker',projectImportPipelineWithJobandVersion])
commandLine 'echo','Testing'
}
If you got this response:
{"status":"success", "exportfile":"export/zip/path.zip"}]}{status=success, job=[{id='JOB_ID', name='JOB_NAME'}], pipeline=[{ name = 'PIPELINE_NAME', id= 'PIPELINE_ID' }]}For every export / import tasks with corresponding task then the build script did succeed.
You then need to check in the saagie platform that the all the job / pipeline that is exported, and the versions for the job or the pipeline is added like it s shown in the job.json/pipeline.json for every job or pipeline in every exported artifact
Enhance the current projectImport to support exports of V1 artifacts including versions. The import should be able to support all versions and import in the same order. i.e if a job in v1 has 3 versions , then the import should also import in the same order and match for the version numbers.