search

matomo-org / matomo

Empowering People Ethically with the leading open source alternative to Google Analytics that gives you full control over your data. Matomo lets you easily collect data from websites & apps and visualise this data and extract insights. Privacy is built-in. Liberating Web Analytics. Star us on Github? +1. And we love Pull Requests!
https://matomo.org/
GNU General Public License v3.0
19.92k stars 2.66k forks source link

[Bug] Archiving Infinite loop in cron tasks #22782

Open reeslo opened 1 day ago

reeslo commented 1 day ago

What happened?

I am experiencing an issue where my Matomo instance is unable to complete its scheduled archiving tasks triggered by cron jobs. The archiving process seems to enter an infinite loop and never finishes.

The issue has significantly affected the performance of my instance and has resulted in resource consumption spikes on my server.

What should happen?

The archiving process should complete successfully without entering an infinite loop. Cron jobs should terminate after processing the scheduled archiving tasks.

How can this be reproduced?

  1. Configure Matomo with the following environment:
    • Cron task for archiving set up as: php -d memory_limit=-1 console core:archive --url=https://xxx.xxx.com/ --matomo-domain=xxx.xxx.com --verbose > /home/xxx.xxx.com/matomo-archive-xxxx.log
    • Instance with a moderate to high amount of traffic.
  2. Trigger the cron task for archiving.
  3. Observe the logs and server behavior:
    • The task never completes.
    • Resource usage increases significantly.
  4. Refer to the attached logs and screenshots for detailed behavior.

Logs download link https://transfert.free.fr/N324JWw

Debug added into ./core/CronArchive/QueueConsumer.php l200 image

config.ini.php

; <?php exit; ?> DO NOT REMOVE THIS LINE
; file automatically generated or modified by Matomo; you can manually override the default values in global.ini.php by redefining them in this file.
[database]
host = "XXXX"
username = "XXXX"
password = "XXXX"
dbname = "XXXX"
tables_prefix = "matomo_"
charset = "utf8mb4"
collation = "utf8mb4_general_ci"
enable_segment_first_table_join_prefix = 1
enable_first_table_join_prefix = 1

[database_reader]
host = "XXXX"
username = "XXXX"
password = "XXXX"
dbname = "XXXX"
aurora_readonly_read_committed = 0

[log]
log_writers[] = "file"
log_writers[] = "screen"
logger_file_path = "/home/XXXX.XXXX.com/matomo.log"

[ChainedCache]
backends[] = "array"
backends[] = "redis"

[RedisCache]
host = "XXXX"
port = XXXX
database = XX

[General]
disable_archiving_segment_for_plugins[]="Funnels"
enable_processing_unique_visitors_year = 1
enable_processing_unique_visitors_range = 1
anonymous_user_enable_use_segments_API = 0
browser_archiving_disabled_enforce = 1
enable_browser_archiving_triggering = 0
enable_sql_optimize_queries = 0
force_ssl = 1
login_allowlist_apply_to_reporting_api_requests = 0
login_allow_logme = 1
enable_load_data_infile = 0
enable_tracking_failures_notification = 0
rearchive_reports_in_past_last_n_months = 3
delete_reports_enable = 1
delete_reports_older_than = 270
instance_id = "XXXX.XXXX.com"
salt = "XXXX"
trusted_hosts[] = "XXXX.XXXX.com"
trusted_hosts[] = "XXXX.XXXX.com"
trusted_hosts[] = "XXXX.XXXX.com"

[Tracker]
ignore_visits_cookie_name = "piwik_ignore"

[Plugins]
Plugins[] = "CoreVue"
Plugins[] = "CorePluginsAdmin"
Plugins[] = "CoreAdminHome"
Plugins[] = "CoreHome"
Plugins[] = "WebsiteMeasurable"
Plugins[] = "IntranetMeasurable"
Plugins[] = "Diagnostics"
Plugins[] = "CoreVisualizations"
Plugins[] = "Proxy"
Plugins[] = "API"
Plugins[] = "Widgetize"
Plugins[] = "Transitions"
Plugins[] = "LanguagesManager"
Plugins[] = "Actions"
Plugins[] = "Dashboard"
Plugins[] = "MultiSites"
Plugins[] = "Referrers"
Plugins[] = "UserLanguage"
Plugins[] = "DevicesDetection"
Plugins[] = "Goals"
Plugins[] = "Ecommerce"
Plugins[] = "SEO"
Plugins[] = "Events"
Plugins[] = "UserCountry"
Plugins[] = "GeoIp2"
Plugins[] = "VisitsSummary"
Plugins[] = "VisitFrequency"
Plugins[] = "VisitTime"
Plugins[] = "VisitorInterest"
Plugins[] = "RssWidget"
Plugins[] = "Feedback"
Plugins[] = "Monolog"
Plugins[] = "Login"
Plugins[] = "TwoFactorAuth"
Plugins[] = "UsersManager"
Plugins[] = "SitesManager"
Plugins[] = "Installation"
Plugins[] = "CoreUpdater"
Plugins[] = "CoreConsole"
Plugins[] = "ScheduledReports"
Plugins[] = "UserCountryMap"
Plugins[] = "Live"
Plugins[] = "PrivacyManager"
Plugins[] = "ImageGraph"
Plugins[] = "Annotations"
Plugins[] = "MobileMessaging"
Plugins[] = "Overlay"
Plugins[] = "SegmentEditor"
Plugins[] = "Insights"
Plugins[] = "Morpheus"
Plugins[] = "Contents"
Plugins[] = "BulkTracking"
Plugins[] = "Resolution"
Plugins[] = "DevicePlugins"
Plugins[] = "Heartbeat"
Plugins[] = "Intl"
Plugins[] = "Marketplace"
Plugins[] = "ProfessionalServices"
Plugins[] = "UserId"
Plugins[] = "CustomJsTracker"
Plugins[] = "Tour"
Plugins[] = "PagePerformance"
Plugins[] = "CustomDimensions"
Plugins[] = "JsTrackerInstallCheck"
Plugins[] = "TagManager"
Plugins[] = "Funnels"
Plugins[] = "MarketingCampaignsReporting"
Plugins[] = "QueuedTracking"

[PluginsInstalled]
PluginsInstalled[] = "Diagnostics"
PluginsInstalled[] = "Login"
PluginsInstalled[] = "CoreAdminHome"
PluginsInstalled[] = "UsersManager"
PluginsInstalled[] = "SitesManager"
PluginsInstalled[] = "Installation"
PluginsInstalled[] = "Monolog"
PluginsInstalled[] = "Intl"
PluginsInstalled[] = "CoreVue"
PluginsInstalled[] = "CorePluginsAdmin"
PluginsInstalled[] = "CoreHome"
PluginsInstalled[] = "WebsiteMeasurable"
PluginsInstalled[] = "IntranetMeasurable"
PluginsInstalled[] = "CoreVisualizations"
PluginsInstalled[] = "Proxy"
PluginsInstalled[] = "API"
PluginsInstalled[] = "Widgetize"
PluginsInstalled[] = "Transitions"
PluginsInstalled[] = "LanguagesManager"
PluginsInstalled[] = "Actions"
PluginsInstalled[] = "Dashboard"
PluginsInstalled[] = "MultiSites"
PluginsInstalled[] = "Referrers"
PluginsInstalled[] = "UserLanguage"
PluginsInstalled[] = "DevicesDetection"
PluginsInstalled[] = "Goals"
PluginsInstalled[] = "Ecommerce"
PluginsInstalled[] = "SEO"
PluginsInstalled[] = "Events"
PluginsInstalled[] = "UserCountry"
PluginsInstalled[] = "GeoIp2"
PluginsInstalled[] = "VisitsSummary"
PluginsInstalled[] = "VisitFrequency"
PluginsInstalled[] = "VisitTime"
PluginsInstalled[] = "VisitorInterest"
PluginsInstalled[] = "RssWidget"
PluginsInstalled[] = "Feedback"
PluginsInstalled[] = "TwoFactorAuth"
PluginsInstalled[] = "CoreUpdater"
PluginsInstalled[] = "CoreConsole"
PluginsInstalled[] = "ScheduledReports"
PluginsInstalled[] = "UserCountryMap"
PluginsInstalled[] = "Live"
PluginsInstalled[] = "PrivacyManager"
PluginsInstalled[] = "ImageGraph"
PluginsInstalled[] = "Annotations"
PluginsInstalled[] = "MobileMessaging"
PluginsInstalled[] = "Overlay"
PluginsInstalled[] = "SegmentEditor"
PluginsInstalled[] = "Insights"
PluginsInstalled[] = "Morpheus"
PluginsInstalled[] = "Contents"
PluginsInstalled[] = "BulkTracking"
PluginsInstalled[] = "Resolution"
PluginsInstalled[] = "DevicePlugins"
PluginsInstalled[] = "Heartbeat"
PluginsInstalled[] = "Marketplace"
PluginsInstalled[] = "ProfessionalServices"
PluginsInstalled[] = "UserId"
PluginsInstalled[] = "CustomJsTracker"
PluginsInstalled[] = "Tour"
PluginsInstalled[] = "PagePerformance"
PluginsInstalled[] = "CustomDimensions"
PluginsInstalled[] = "QueuedTracking"
PluginsInstalled[] = "TagManager"
PluginsInstalled[] = "MarketingCampaignsReporting"
PluginsInstalled[] = "DBStats"
PluginsInstalled[] = "MobileAppMeasurable"
PluginsInstalled[] = "JsTrackerInstallCheck"
PluginsInstalled[] = "Funnels"

[Login]
whitelisteBruteForceIps[] = "XXXX"

[QueuedTracking]
log_failed_tracking_request_body = 1
notify_queue_threshold_emails[] = "XXXX@XXXX.com"
notify_queue_threshold_single_queue = 2500

[Funnels]
funnels_num_max_rows_in_actions = 100
funnels_num_max_rows_in_referrers = 50
funnels_num_max_rows_populate_at_once = 60000

Matomo version

5.1.2

PHP version

8.1

Server operating system

Centos7

What browsers are you seeing the problem on?

No response

Computer operating system

No response

Relevant log output

No response

Validations