Closed HungryHowies closed 2 weeks ago
Debug trace stack
root@zitadel-build:/usr/local/bin# zitadel mirror --system --replace --config /usr/local/bin/config.yaml --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode external
INFO[0000] assets migrated caller="/home/zitadel-2.57.0/cmd/mirror/system.go:92" count=3 took=72.896224ms
INFO[0000] encryption keys migrated caller="/home/zitadel-2.57.0/cmd/mirror/system.go:138" count=10 took=17.891263ms
INFO[0001] auth requests migrated caller="/home/zitadel-2.57.0/cmd/mirror/auth.go:90" count=1105 took=338.429087ms
INFO[0001] start event migration caller="/home/zitadel-2.57.0/cmd/mirror/event_store.go:97" from=1.721958713263689e+09 to=1.7223019667620401e+09
INFO[0001] events migrated caller="/home/zitadel-2.57.0/cmd/mirror/event_store.go:180" count=0 took=35.371683ms
INFO[0001] unique constraints migrated caller="/home/zitadel-2.57.0/cmd/mirror/event_store.go:249" count=180 took=7.790492ms
INFO[0001] projections executed caller="/home/zitadel-2.57.0/cmd/mirror/projections.go:250" took=46.352642ms
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2 source=2 table=adminapi.styling2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=adminapi.locks
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=adminapi.current_sequences
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=adminapi.failed_events
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=adminapi.styling
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.user_external_idps2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-12 source=12 table=auth.users2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.locks
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.current_sequences
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.users
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-114 source=114 table=auth.user_sessions
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.user_external_idps
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.failed_events
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-32 source=32 table=auth.tokens
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.org_project_mapping
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.idp_providers
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.idp_configs
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=1105 source=1105 table=auth.auth_requests
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.refresh_tokens
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.org_project_mapping2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.idp_providers2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=auth.idp_configs2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-10 source=10 table=auth.users3
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2413 source=2413 table=eventstore.events2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-119 source=119 table=eventstore.fields
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=180 source=180 table=eventstore.unique_constraints
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.current_sequences
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-41 source=41 table=projections.failed_events2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-81 source=81 table=projections.current_states
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.privacy_policies4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.domain_policies2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2 source=2 table=projections.label_policies3
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idps3
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idps3_oidc_config
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idps3_jwt_config
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_oauth2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_oidc
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_jwt
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_azure
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_github_enterprise
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_gitlab
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_gitlab_self_hosted
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_google
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-8 source=8 table=projections.apps7_saml_configs
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-5 source=5 table=projections.apps7_oidc_configs
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-10 source=10 table=projections.users13_notifications
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.login_names3_policies
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-14 source=14 table=projections.users13
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-10 source=10 table=projections.users13_humans
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-4 source=4 table=projections.users13_machines
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-6 source=6 table=projections.org_members4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-4 source=4 table=projections.instance_domains
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2 source=2 table=projections.instance_members4
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.sms_configs2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.sms_configs2_twilio
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-9 source=9 table=projections.secret_generators2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.smtp_configs2
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=projections.instances
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=projections.instances
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.instances
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-3 source=3 table=projections.keys4_certificate
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.security_policies2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.limits
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.restrictions2
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=projections.system_features
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=projections.system_features
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.system_features
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2 source=2 table=projections.instance_features2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.targets1
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.user_schemas
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.executions1
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.executions1_targets
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-14 source=14 table=projections.login_names3_users
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-5 source=5 table=projections.login_names3_domains
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.locks
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.failed_events
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-4 source=4 table=projections.orgs1
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.org_metadata2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.actions3
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_github
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-4 source=4 table=projections.flow_triggers3
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-12 source=12 table=projections.projects4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.password_complexity_policies2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.password_age_policies2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.lockout_policies3
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-10 source=10 table=projections.project_grants4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-8 source=8 table=projections.project_roles4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-5 source=5 table=projections.org_domains2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-3 source=3 table=projections.login_policies5
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_ldap2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_apple
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_templates6_saml
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-16 source=16 table=projections.apps7
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.project_grant_members4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-4 source=4 table=projections.authn_keys2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-3 source=3 table=projections.apps7_api_configs
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-7 source=7 table=projections.personal_access_tokens3
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-16 source=16 table=projections.user_grants5
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-5 source=5 table=projections.user_metadata5
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-3 source=3 table=projections.user_auth_methods4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.oidc_settings2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-1 source=1 table=projections.mail_templates2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-12 source=12 table=projections.message_texts2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_user_links3
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.idp_login_policy_links5
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-79 source=79 table=projections.custom_texts2
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-15 source=15 table=projections.project_members4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-6 source=6 table=projections.milestones
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.quotas
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.quotas_periods
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.quotas_notifications
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.notification_providers
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-90 source=90 table=projections.keys4
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-90 source=90 table=projections.keys4_private
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-90 source=90 table=projections.keys4_public
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-2 source=2 table=projections.notification_policies
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.device_auth_requests2
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.sessions8
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=projections.auth_requests
INFO[0001] unequal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:55" dest=0 diff=-14 source=14 table=projections.login_names3
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=system.encryption_keys
ERRO[0001] unexpected query error caller="/home/zitadel-2.57.0/internal/database/database.go:88" error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)"
ERRO[0001] unable to count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:108" db=zitadel error="ERROR: column \"instance_id\" does not exist (SQLSTATE 42703)" table=system.encryption_keys
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=0 source=0 table=system.encryption_keys
DEBU[0001] equal count caller="/home/zitadel-2.57.0/cmd/mirror/verify.go:52" dest=3 source=3 table=system.assets
Using my new configuration file (defaults.yaml) which has setting for PostgreSQL, for the first two commands.
Last command is the config.yaml file that has configurations for destination and source database. which was shown in the original setup.