search

jenkinsci / kubernetes-operator

Kubernetes native Jenkins Operator
https://jenkinsci.github.io/kubernetes-operator
Other
604 stars 237 forks source link

Helm install doesn't create necessary role permissions #424

Closed bainss closed 3 years ago

bainss commented 4 years ago

Expected Behavior

Kubernetes pods created on demand by the kubernetes plugin

Actual Behavior

The following in the logs and no worker pod created

io.fabric8.kubernetes.client.KubernetesClientException: events is forbidden: User "system:serviceaccount:jenkins:jenkins-operator-jenkins" cannot watch resource "events" in API group "" in the namespace "jenkins"

Steps to Reproduce the Problem

  1. Create groovy Config map

    apiVersion: v1
kind: ConfigMap
metadata:
name: jenkins-operator-groovy-configuration
data:
1-configure-theme.groovy: |
import jenkins.*
import jenkins.model.*
import hudson.*
import hudson.model.*
import org.jenkinsci.plugins.simpletheme.ThemeElement
import org.jenkinsci.plugins.simpletheme.CssTextThemeElement
import org.jenkinsci.plugins.simpletheme.CssUrlThemeElement

Jenkins jenkins = Jenkins.getInstance()

def decorator = Jenkins.instance.getDescriptorByType(org.codefirst.SimpleThemeDecorator.class)

List<ThemeElement> configElements = new ArrayList<>();
configElements.add(new CssTextThemeElement("DEFAULT"));
configElements.add(new CssUrlThemeElement("https://tobix.github.io/jenkins-neo2-theme/dist/neo-light.css"));
decorator.setElements(configElements);
decorator.save();

jenkins.save()

  2. Create jcasc Config map

apiVersion: v1
kind: ConfigMap
metadata:
  name: jenkins-operator-jcasc-configuration
data:
  configuration-as-code.yaml: |
    jenkins:
      systemMessage: "Continuous Integration. Please see https://github.com/polarpoint-io/DO/jenkins-operator-jcasc for more details."
      numExecutors: 12
      clouds:
      - kubernetes:
         containerCap: 10
         containerCapStr: "10"
         maxRequestsPerHost: 32
         maxRequestsPerHostStr: "32"
         name: "kubernetes"
         templates:
         - hostNetwork: false
           label: "gradle-6"
           name: "gradle-6"
           yaml: |-
             apiVersion: v1
             kind: Pod
             spec:
               containers:
               - name: gradle-6-0
                 image: amd64/gradle:6.4.0-jdk11
                 command:
                 - cat
                 tty: true
           yamlMergeStrategy: "override"
           yamls:
           - |-
             apiVersion: v1
             kind: Pod
             spec:
               containers:
               - name: gradle-6
                 image: amd64/gradle:6.4.0-jdk11
                 command:
                 - cat
                 tty: true
    unclassified:
     gitscm:
       globalConfigName: jenkins
       globalConfigEmail: jenkins@domain.local
       createAccountBasedOnEmail: true
     globalLibraries:
           libraries:
             - name: "pipeline-library"
               retriever:
                 modernSCM:
                   scm:
                     git:
                       remote: "https://github.com/polarpoint-io/groovy-jenkins-shared-library"
                       credentialsId: "service-github-token"

3.

Additional Info

Helm values file

# Jenkins Operator Helm chart

# Jenkins instance configuration
jenkins:
  # enabled can enable or disable the Jenkins instance
  # Set to false if you have configured CR already and/or you want to deploy an operator only
  enabled: true

  # apiVersion is the version of the CR manifest
  # The recommended and default value is "jenkins.io/v1alpha2"
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/v0.1.x/migration-guide-v1alpha1-to-v1alpha2/ for more migration guide
  apiVersion: jenkins.io/v1alpha2

  # name of resource
  # The pod name will be jenkins-<name> (name will be set as suffix)
  name: jenkins

  # namespace is the namespace where the resources will be deployed
  # It's not recommended to use default namespace
  # Create new namespace for jenkins (called e.g. jenkins)
  namespace: jenkins

  # labels are injected into metadata labels field
  labels: {}

  # image is the name (and tag) of the Jenkins instance
  # Default: jenkins/jenkins:lts
  # It's recommended to use LTS (tag: "lts") version
  image: jenkins/jenkins:lts

  # env contains jenkins container environment variables
  env: []

  # imagePullPolicy defines policy for pulling images
  imagePullPolicy: Always

  # priorityClassName indicates the importance of a Pod relative to other Pods
  # See: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
  priorityClassName: ""

  # disableCSRFProtection can enable or disable operator built-in CSRF protection
  # Set it to true if you are using OpenShift Jenkins Plugin
  # See https://github.com/jenkinsci/kubernetes-operator/pull/193 for more info
  disableCSRFProtection: false

  # imagePullSecrets is used if you want to pull images from private repository
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/configuration/#pulling-docker-images-from-private-repositories for more info
  imagePullSecrets: []

  # notifications is feature that notify user about Jenkins reconcilation status
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/notifications/ for more info
  notifications:
  - level: info
    verbose: true
    name: nslack
    slack:
      webHookURLSecretKeySelector:
        secret:
          name: jenkins-operator-notification-data
        key: slack-webhook-url
  - level: info
    verbose: true
    name: nteams
    teams:
      webHookURLSecretKeySelector:
        secret:
          name: jenkins-operator-notification-data
        key: teams-webhook-url
  # basePlugins are plugins installed and required by the operator
  # Shouldn't contain plugins defined by user
  # You can change their versions here
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/customization/#install-plugins for more details
  #
  # Example:
  #
  # basePlugins:
  # - name: kubernetes
  #   version: 1.18.3
  # - name: workflow-job
  #   version: "2.34"
  # - name: workflow-aggregator
  #   version: "2.6"
  # - name: git
  #   version: 3.12.0
  # - name: job-dsl
  #   version: "1.76"
  # - name: configuration-as-code
  #   version: "1.29"
  # - name: kubernetes-credentials-provider
  #   version: 0.12.1
  basePlugins:
  - name: kubernetes
    version: "1.26.1"
  - name: workflow-job
    version: "2.39"
  - name: workflow-aggregator
    version: "2.6"
  - name: git
    version: "4.2.2"
  - name: job-dsl
    version: "1.77"
  - name: configuration-as-code
    version: "1.40"
  - name: kubernetes-credentials-provider
    version: "0.15"
  # plugins are plugins required by the user
  # You can define plugins here
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/customization/#install-plugins for more details
  #
  # Example:
  #
  # plugins:
  # - name: simple-theme-plugin
  #   version: 0.5.1
  plugins:
  - name: ace-editor
    version: "1.1"
  - name: analysis-core
    version: "1.96"
  - name: analysis-model-api
    version: "8.1.3"
  - name: ansicolor
    version: "0.7.0"
  - name: antisamy-markup-formatter
    version: "2.0"
  - name: apache-httpcomponents-client-4-api
    version: "4.5.10-2.0"
  - name: authentication-tokens
    version: "1.3"
  - name: badge
    version: "1.8"
  - name: block-queued-job
    version: "0.2.0"
  - name: blueocean
    version: "1.23.2"
  - name: blueocean-autofavorite
    version: "1.2.4"
  - name: blueocean-bitbucket-pipeline
    version: "1.23.2"
  - name: blueocean-commons
    version: "1.23.2"
  - name: blueocean-config
    version: "1.23.2"
  - name: blueocean-core-js
    version: "1.23.2"
  - name: blueocean-dashboard
    version: "1.23.2"
  - name: blueocean-display-url
    version: "2.3.1"
  - name: blueocean-events
    version: "1.23.2"
  - name: blueocean-executor-info
    version: "1.23.2"
  - name: blueocean-git-pipeline
    version: "1.23.2"
  - name: blueocean-github-pipeline
    version: "1.23.2"
  - name: blueocean-i18n
    version: "1.23.2"
  - name: blueocean-jira
    version: "1.23.2"
  - name: blueocean-jwt
    version: "1.23.2"
  - name: blueocean-personalization
    version: "1.23.2"
  - name: blueocean-pipeline-api-impl
    version: "1.23.2"
  - name: blueocean-pipeline-editor
    version: "1.23.2"
  - name: blueocean-pipeline-scm-api
    version: "1.23.2"
  - name: blueocean-rest
    version: "1.23.2"
  - name: blueocean-rest-impl
    version: "1.23.2"
  - name: blueocean-web
    version: "1.23.2"
  - name: bootstrap4-api
    version: "4.5.0-1"
  - name: bouncycastle-api
    version: "2.18"
  - name: branch-api
    version: "2.5.6"
  - name: build-monitor-plugin
    version: "1.12+build.201809061734"
  - name: checkstyle
    version: "4.0.0"
  - name: cloudbees-bitbucket-branch-source
    version: "2.8.0"
  - name: cloudbees-folder
    version: "6.14"
  - name: code-coverage-api
    version: "1.1.6"
  - name: command-launcher
    version: "1.2"
  - name: credentials
    version: "2.3.9"
  - name: credentials-binding
    version: "1.23"
  - name: dashboard-view
    version: "2.12"
  - name: data-tables-api
    version: "1.10.21-2"
  - name: dependency-check-jenkins-plugin
    version: "5.1.1"
  - name: dependency-track
    version: "2.3.0"
  - name: display-url-api
    version: "2.3.2"
  - name: docker-commons
    version: "1.16"
  - name: docker-workflow
    version: "1.23"
  - name: durable-task
    version: "1.34"
  - name: echarts-api
    version: "4.7.0-4"
  - name: external-monitor-job
    version: "1.7"
  - name: favorite
    version: "2.3.2"
  - name: findbugs
    version: "5.0.0"
  - name: font-awesome-api
    version: "5.13.0-1"
  - name: forensics-api
    version: "0.7.0"
  - name: git-client
    version: "3.2.1"
  - name: git-server
    version: "1.9"
  - name: github
    version: "1.30.0"
  - name: github-api
    version: "1.114.2"
  - name: github-branch-source
    version: "2.8.2"
  - name: github-pullrequest
    version: "0.2.8"
  - name: greenballs
    version: "1.15"
  - name: handlebars
    version: "1.1.1"
  - name: handy-uri-templates-2-api
    version: "2.1.8-1.0"
  - name: htmlpublisher
    version: "1.23"
  - name: icon-shim
    version: "2.0.3"
  - name: ignore-committer-strategy
    version: "1.0.4"
  - name: jackson2-api
    version: "2.11.0"
  - name: jacoco
    version: "3.0.6"
  - name: javadoc
    version: "1.5"
  - name: jdk-tool
    version: "1.4"
  - name: jenkins-design-language
    version: "1.23.2"
  - name: jira
    version: "3.1.1"
  - name: job-dsl
    version: "1.77"
  - name: jquery-detached
    version: "1.2.1"
  - name: jquery3-api
    version: "3.5.1-1"
  - name: jsch
    version: "0.1.55.2"
  - name: junit
    version: "1.29"
  - name: kubernetes-client-api
    version: "4.9.2-2"
  - name: kubernetes-credentials
    version: "0.7.0"
  - name: ldap
    version: "1.24"
  - name: lockable-resources
    version: "2.8"
  - name: mailer
    version: "1.32"
  - name: matrix-auth
    version: "2.6.1"
  - name: matrix-project
    version: "1.16"
  - name: maven-plugin
    version: "3.6"
  - name: mercurial
    version: "2.10"
  - name: momentjs
    version: "1.1.1"
  - name: okhttp-api
    version: "3.12.12.2"
  - name: pam-auth
    version: "1.6"
  - name: performance
    version: "3.17"
  - name: pipeline-build-step
    version: "2.12"
  - name: pipeline-github-lib
    version: "1.0"
  - name: pipeline-graph-analysis
    version: "1.10"
  - name: pipeline-input-step
    version: "2.11"
  - name: pipeline-milestone-step
    version: "1.3.1"
  - name: pipeline-model-api
    version: "1.7.0"
  - name: pipeline-model-definition
    version: "1.7.0"
  - name: pipeline-model-extensions
    version: "1.7.0"
  - name: pipeline-rest-api
    version: "2.13"
  - name: pipeline-stage-step
    version: "2.3"
  - name: pipeline-stage-tags-metadata
    version: "1.7.0"
  - name: pipeline-stage-view
    version: "2.13"
  - name: plain-credentials
    version: "1.7"
  - name: plugin-util-api
    version: "1.2.2"
  - name: popper-api
    version: "1.16.0-6"
  - name: pubsub-light
    version: "1.13"
  - name: scm-api
    version: "2.6.3"
  - name: script-security
    version: "1.73"
  - name: simple-theme-plugin
    version: "0.6"
  - name: slack
    version: "2.40"
  - name: snakeyaml-api
    version: "1.26.4"
  - name: sonar
    version: "2.11"
  - name: sonar-quality-gates
    version: "1.3.1"
  - name: sse-gateway
    version: "1.23"
  - name: ssh-credentials
    version: "1.18.1"
  - name: structs
    version: "1.20"
  - name: theme-manager
    version: "0.1"
  - name: timestamper
    version: "1.11.3"
  - name: token-macro
    version: "2.12"
  - name: trilead-api
    version: "1.0.8"
  - name: variant
    version: "1.3"
  - name: warnings-ng
    version: "8.1.0"
  - name: windows-slaves
    version: "1.6"
  - name: workflow-api
    version: "2.40"
  - name: workflow-basic-steps
    version: "2.20"
  - name: workflow-cps
    version: "2.80"
  - name: workflow-cps-global-lib
    version: "2.16"
  - name: workflow-cps-global-lib-http
    version: "1.5.0"
  - name: workflow-durable-task-step
    version: "2.35"
  - name: workflow-multibranch
    version: "2.21"
  - name: workflow-scm-step
    version: "2.11"
  - name: workflow-step-api
    version: "2.22"
  - name: workflow-support
    version: "3.5"
  - name: zap-pipeline
    version: "1.9"
  - name: zapper
    version: "1.0.7"

  # seedJobs is placeholder for jenkins seed jobs
  # For seed job creation tutorial, check https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/configuration/#prepare-job-definitions-and-pipelines
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/configuration/#configure-seed-jobs for additional info
  # Example:
  #
  seedJobs:
   - id: jenkins-operator
     targets: "*/**.groovy"
     description: "Jenkins Operator seeding repository"
     repositoryBranch: master
     repositoryUrl: https://github.com/polarpoint-io/groovy-jenkins-pipelines.git
  # Resource limit/request for Jenkins
  # See https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ for details
  resources:
    limits:
      cpu: 4
      memory: 8Gi
    requests:
      cpu: 2
      memory: 2Gi

  # volumes used by Jenkins
  # By default, we are only using backup
  volumes:
    - name: backup # PVC volume where backups will be stored
      persistentVolumeClaim:
        claimName: jenkins-backup

  # securityContext for pod
  securityContext:
    runAsUser: 1000
    fsGroup: 1000

  # http Jenkins service
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/schema/#github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Service for details
  #service:
  # slave Jenkins service
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/schema/#github.com/jenkinsci/kubernetes-operator/pkg/apis/jenkins/v1alpha2.Service for details
  #slaveService:

  # backup is section for configuring operator's backup feature
  # By default backup feature is enabled and pre-configured
  # This section simplifies the configuration described here: https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/configure-backup-and-restore/
  # For customization tips see https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/custom-backup-and-restore/
  backup:
    # enabled is enable/disable switch for backup feature
    # By default the feature is enabled
    enabled: true

    # image used by backup feature
    # By default using prebuilt backup PVC image by VirtusLab
    image: virtuslab/jenkins-operator-backup-pvc:v0.0.8

    # containerName is backup container name
    containerName: backup

    # interval defines how often make backup in seconds
    interval: 30

    # makeBackupBeforePodDeletion when enabled will make backup before pod deletion
    makeBackupBeforePodDeletion: true

    # backupCommand is backup container command
    backupCommand:
      - /home/user/bin/backup.sh

    # restoreCommand is backup restore command
    restoreCommand:
      - /home/user/bin/restore.sh

    # pvc is Persistent Volume Claim Kubernetes resource
    pvc:
      # enabled is enable/disable switch for PVC
      enabled: true

      # size is size of PVC
      size: 50Gi

      # className is storageClassName for PVC
      # See https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1 for more details
      className: ""

    # env contains container environment variables
    # PVC backup provider handles these variables:
    # BACKUP_DIR - path for storing backup files (default: "/backup")
    # JENKINS_HOME - path to jenkins home (default: "/jenkins-home")
    # BACKUP_COUNT - define how much recent backups will be kept
    env:
      - name: BACKUP_DIR
        value: /backup
      - name: JENKINS_HOME
        value: /jenkins-home
      - name: BACKUP_COUNT
        value: "3" # keep only the 3 most recent backups

    # volumeMounts holds the mount points for volumes
    volumeMounts:
      - name: jenkins-home
        mountPath: /jenkins-home # Jenkins home volume
      - mountPath: /backup # backup volume
        name: backup

  # configuration is section where we can configure Jenkins instance
  # See https://jenkinsci.github.io/kubernetes-operator/docs/getting-started/latest/customization/ for details
  configuration:
    configurationAsCode:
    - configMapName: jenkins-operator-jcasc-configuration
    groovyScripts:
    - configMapName: jenkins-operator-groovy-configuration
    # secretRefName of existing secret (previously created)
    secretRefName: ""

    # secretData creates new secret if secretRefName is empty and fills with data provided in secretData
    secretData: {}

# operator is section for configuring operator deployment
operator:
  replicaCount: 1

  # image is the name (and tag) of the Jenkins Operator image
  image: virtuslab/jenkins-operator:v0.4.0

  # imagePullPolicy defines policy for pulling images
  imagePullPolicy: IfNotPresent

  # imagePullSecrets is used if you want to pull images from private repository
  imagePullSecrets: []

  # nameOverride overrides the app name
  nameOverride: ""

  # fullnameOverride overrides the deployment name
  fullnameOverride: ""

  resources: {}
  nodeSelector: {}
  tolerations: []
  affinity: {}
k version
Client Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.2", GitCommit:"59603c6e503c87169aea6106f57b9f242f64df89", GitTreeState:"clean", BuildDate:"2020-01-18T23:30:10Z", GoVersion:"go1.13.5", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.2", GitCommit:"59603c6e503c87169aea6106f57b9f242f64df89", GitTreeState:"clean", BuildDate:"2020-01-18T23:22:30Z", GoVersion:"go1.13.5", Compiler:"gc", Platform:"linux/amd64"}

0.4.0

I did attempt to make the changes to the helm template (copied from the all in one yaml file)

cat  jenkins-operator/templates/role.yaml
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-operator-jenkins
rules:
  - apiGroups:
      - ""
    resources:
      - services
      - configmaps
      - secrets
      - serviceaccounts
      - events
    verbs:
      - get
      - create
      - update
      - list
      - watch
  - apiGroups:
      - apps
    resources:
      - deployments
      - daemonsets
      - replicasets
      - statefulsets
      - events
    verbs:
      - '*'
  - apiGroups:
      - rbac.authorization.k8s.io
    resources:
      - roles
      - rolebindings
    verbs:
      - create
      - update
      - list
      - watch
      - events
  - apiGroups:
      - ""
    resources:
      - pods/portforward
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - pods/log
    verbs:
      - get
      - list
      - watch
      - events
  - apiGroups:
      - ""
    resources:
      - pods
      - pods/exec
    verbs:
      - "*"
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - watch
      - list
      - create
      - patch
  - apiGroups:
      - apps
    resourceNames:
      - jenkins-operator
    resources:
      - deployments/finalizers
    verbs:
      - update
  - apiGroups:
      - jenkins.io
    resources:
      - '*'
    verbs:
      - '*'
  - apiGroups:
      - ""
    resources:
      - persistentvolumeclaims
    verbs:
      - get
      - list
      - watch
      - events

and apply helm as follows

helm upgrade -i jenkins-operator-release ./jenkins-operator  --values jenkins-operator-values.yaml --namespace jenkins
SylwiaBrant commented 3 years ago

Hello, we're planning a new release soon which will fix this.

tomaszsek commented 3 years ago

Fixed in https://github.com/jenkinsci/kubernetes-operator/releases/tag/v0.5.0