search

PyO3 / maturin

Build and publish crates with pyo3, cffi and uniffi bindings as well as rust binaries as python packages
https://maturin.rs
Apache License 2.0
3.88k stars 269 forks source link

Add support for jenkins as CI provider #1775

Open cojmeister opened 1 year ago

cojmeister commented 1 year ago

Hey, I love this tool! I want to add support for Jenkins as a CI provider. I can do it, but I need some help/mentoring in order to properly define a CI. From what I see in the current CI is:

I also see that there is extensive use of maturin-action - which isn't one to one with maturin cli tool.

As a first draft I've created the following workflow. It first checks out. Then sets up python and builds parallel for each agent (windows ubuntu and macos), and then stashes the result. Finally it releases a version to pypi.

pipeline {
    agent none
    stages {
        stage('Checkout') {
            agent any
            steps {
                checkout scm
            }
        }
        stage('Build') {
            failFast true
            parallel {
                stage('windows') {
                    agent {
                        label "windows-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip install maturin'
                            }
                        }
                        stage('Build and stash') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x64', 'x86'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-pc-windows-gnu"
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'windows'
                        }
                    }
                }
                stage('linux') {
                    agent {
                        label "ubuntu-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip3.10 install maturin'
                            }
                        }
                        stage('Build and stash') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x86_64', 'x86', 'aarch64', 'armv7', 's390x', 'ppc64le'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        // This won't work for all given archs - need to understand how to run with maturin
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-unknown-linux-gnu"
                                        // Need to add sccache and manylinux - currently getting error on sccache (locally)
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'ubuntu'
                        }
                    }
                }
                stage('macos') {
                    agent {
                        label "macos-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip3.10 install maturin'
                            }
                        }
                        stage('Build') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x86_64', 'aarch64'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-apple-darwin"
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'macos'
                        }
                    }
                }
                stage('sdist') {
                    agent {
                        label 'ubuntu-latest'
                    }
                    stages {
                        stage('build sdist') {
                            // For some reason this is causing an incredibly large file (locally)
                            sh 'maturin build --sdist  --out dist' 
                        }
                        stage('stash sdist') {
                            stash includes: 'dist/**/*', name: 'sdist'
                        }
                    }
                }
            }
        }
        stage('Release') {
            when {} // Need to better understand when to release
            agent {
                label "ubuntu-latest"
            }
            environment { 
                MATURIN_PYPI_TOKEN = credentials('my-pypi-api-token') 
            }
            steps {
                unstash 'windows'
                unstash 'macos'
                unstash 'linux'
                unstash 'sdist'
                // Need to deal with Pypi token
                sh 'maturin upload --non-interactive --skip-existing ./dist/*'
            }
        }
    }
}

What do you think? Is this build good? Should I add our remove anything?

messense commented 1 year ago

Looks reasonable, but manylinux is a must have feature.

cojmeister commented 1 year ago

@messense can you explain what the feature does? I'm not understanding it from the documentation, nor from the code

messense commented 1 year ago

@cojmeister https://github.com/pypa/manylinux

cojmeister commented 1 year ago

@messense ok What I don't understand is the logic in the maturin-action. I mean, shouldn't the action replicate the build command in the CLI? In any case, when I set manylinux, I'm basically accessing this HashMap?

const DEFAULT_CONTAINERS: Record<string, Record<string, string>> = {
  'x86_64-unknown-linux-gnu': {
    auto: 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2010': 'quay.io/pypa/manylinux2010_x86_64:latest',
    '2_12': 'quay.io/pypa/manylinux2010_x86_64:latest',
    '2014': 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2_17': 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2_24': 'quay.io/pypa/manylinux_2_24_x86_64:latest',
    '2_28': 'quay.io/pypa/manylinux_2_28_x86_64:latest'
  },
  'x86_64-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl'
  },
  'i686-unknown-linux-gnu': {
    auto: 'quay.io/pypa/manylinux2014_i686:latest',
    '2010': 'quay.io/pypa/manylinux2010_i686:latest',
    '2_12': 'quay.io/pypa/manylinux2010_i686:latest',
    '2014': 'quay.io/pypa/manylinux2014_i686:latest',
    '2_17': 'quay.io/pypa/manylinux2014_i686:latest',
    '2_24': 'quay.io/pypa/manylinux_2_24_i686:latest'
  },
  'i686-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl'
  },
  'aarch64-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2_24': 'messense/manylinux_2_24-cross:aarch64',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:aarch64'
  },
  'aarch64-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl'
  },
  'armv7-unknown-linux-gnueabihf': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2_24': 'messense/manylinux_2_24-cross:armv7',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:armv7'
  },
  'armv7-unknown-linux-musleabihf': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf'
  },
  'powerpc64-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:ppc64',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64'
  },
  'powerpc64le-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2_24': 'messense/manylinux_2_24-cross:ppc64le',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:ppc64le'
  },
  'powerpc64le-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl'
  },
  's390x-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2_24': 'messense/manylinux_2_24-cross:s390x',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:s390x'
  }
}

I'd like to think that copying this as is is not a smart idea, any better recommendations?

messense commented 1 year ago

I mean, shouldn't the action replicate the build command in the CLI?

Yes but the build command needs to run in a manylinux compatible environment, hence the docker containers.