sosreport / sos

A unified tool for collecting system logs and other debug information
http://sos.rtfd.org
GNU General Public License v2.0
514 stars 540 forks source link

[Ceph]Collection of logs present in /var/log/ceph folder #3809

Open AmarnatReddy opened 1 month ago

AmarnatReddy commented 1 month ago

Collection of logs present in /var/log/ceph folder

I see below commands getting executed and outputs getting collected with respect to ceph. Is there a option to collect ceph logs present in /var/log/ceph folder

[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# ls -lrt | grep ceph
drwx------. 2 root root   21 Oct 15 04:16 ceph_common
drwx------. 2 root root 8192 Oct 15 04:16 ceph_mds
drwx------. 3 root root 4096 Oct 15 04:17 ceph_mgr
drwx------. 3 root root 4096 Oct 15 04:18 ceph_mon
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# s -lrt ceph_common/
-bash: s: command not found
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# ls -lrt ceph_common/
total 4
-rw-r--r--. 1 root root 86 Oct 15 04:10 ceph_-v
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# ls -lrt ceph_mds
total 96
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_cache_status
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_client_ls
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_config_diff
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_config_show
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_counter_dump
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_counter_schema
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_damage_ls
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_loads
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_tree
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_blocked_ops
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_historic_ops
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_historic_ops_by_duration
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_mempools
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_dump_ops_in_flight
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_get_subtrees
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_objecter_requests
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_ops
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_perf_dump
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_perf_histogram_dump
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_perf_histogram_schema
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_perf_schema
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_session_ls
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_status
-rw-r--r--. 1 root root 76 Oct 15 04:10 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mds.cephfs_upgrade_mirror.ceph1-amk-mirror-9tdhoy-node1-installer.dehcgu.asok_version
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# ls -lrt ceph_mgr
total 616
-rw-r--r--. 1 root root    333 Oct 15 04:10 ceph_balancer_status
-rw-r--r--. 1 root root    158 Oct 15 04:10 ceph_healthcheck_history_ls
-rw-r--r--. 1 root root   4520 Oct 15 04:10 ceph_log_last_cephadm
-rw-r--r--. 1 root root 494613 Oct 15 04:10 ceph_mgr_dump
-rw-r--r--. 1 root root   2115 Oct 15 04:10 ceph_mgr_metadata
-rw-r--r--. 1 root root   1332 Oct 15 04:10 ceph_mgr_module_ls
-rw-r--r--. 1 root root    134 Oct 15 04:10 ceph_mgr_stat
-rw-r--r--. 1 root root     99 Oct 15 04:10 ceph_mgr_versions
-rw-r--r--. 1 root root    670 Oct 15 04:10 ceph_orch_host_ls
-rw-r--r--. 1 root root   3458 Oct 15 04:11 ceph_orch_device_ls
-rw-r--r--. 1 root root    973 Oct 15 04:11 ceph_orch_ls
-rw-r--r--. 1 root root    850 Oct 15 04:11 ceph_orch_ls_--export
-rw-r--r--. 1 root root   5304 Oct 15 04:11 ceph_orch_ps
-rw-r--r--. 1 root root     64 Oct 15 04:11 ceph_orch_status_--detail
-rw-r--r--. 1 root root    175 Oct 15 04:11 ceph_orch_upgrade_status
drwx------. 2 root root   4096 Oct 15 04:11 json_output
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_config_diff
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_config_show
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_counter_dump
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_counter_schema
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_dump_cache
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_dump_mempools
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_dump_osd_network
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_mds_requests
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_mds_sessions
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_objecter_requests
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_perf_dump
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_perf_histogram_dump
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_perf_histogram_schema
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_perf_schema
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_status
-rw-r--r--. 1 root root     76 Oct 15 04:11 ceph_daemon_.var.run.ceph.935cb2b8-87e5-11ef-8bd3-fa163e370153.ceph-mgr.ceph1-amk-mirror-9tdhoy-node1-installer.kzebfr.asok_version
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# ls -lrt ceph_mon
total 5408
-rw-r--r--. 1 root root       0 Oct 15 04:10 ceph_crash_ls
-rw-r--r--. 1 root root  150365 Oct 15 04:10 ceph_report
-rw-r--r--. 1 root root    2743 Oct 15 04:10 ceph_config_dump
-rw-r--r--. 1 root root     259 Oct 15 04:10 ceph_config_generate-minimal-conf
-rw-r--r--. 1 root root    2043 Oct 15 04:10 ceph_config_log
-rw-r--r--. 1 root root      19 Oct 15 04:10 ceph_crash_stat
-rw-r--r--. 1 root root     949 Oct 15 04:10 ceph_features
-rw-r--r--. 1 root root      10 Oct 15 04:10 ceph_health_detail
-rw-r--r--. 1 root root     140 Oct 15 04:10 ceph_insights
-rw-r--r--. 1 root root 2444622 Oct 15 04:11 ceph_log_last_10000_debug_audit
-rw-r--r--. 1 root root 2090582 Oct 15 04:11 ceph_log_last_10000_debug_cluster
-rw-r--r--. 1 root root  494613 Oct 15 04:11 ceph_mgr_dump
-rw-r--r--. 1 root root    2115 Oct 15 04:11 ceph_mgr_metadata
-rw-r--r--. 1 root root    1332 Oct 15 04:11 ceph_mgr_module_ls
-rw-r--r--. 1 root root      48 Oct 15 04:11 ceph_mgr_services
-rw-r--r--. 1 root root      99 Oct 15 04:11 ceph_mgr_versions
-rw-r--r--. 1 root root     472 Oct 15 04:11 ceph_mon_stat
-rw-r--r--. 1 root root    9646 Oct 15 04:11 ceph_osd_crush_dump
-rw-r--r--. 1 root root     687 Oct 15 04:11 ceph_osd_crush_show-tunables
-rw-r--r--. 1 root root    2079 Oct 15 04:11 ceph_osd_crush_tree_--show-shadow
-rw-r--r--. 1 root root       8 Oct 15 04:11 ceph_osd_erasure-code-profile_ls
-rw-r--r--. 1 root root   32943 Oct 15 04:11 ceph_osd_metadata
-rw-r--r--. 1 root root    1763 Oct 15 04:11 ceph_quorum_status
-rw-r--r--. 1 root root     733 Oct 15 04:11 ceph_versions
-rw-r--r--. 1 root root      74 Oct 15 04:11 ceph-disk_list
-rw-r--r--. 1 root root    1144 Oct 15 04:11 ceph_device_ls
-rw-r--r--. 1 root root    1302 Oct 15 04:11 ceph_df_detail
-rw-r--r--. 1 root root     726 Oct 15 04:11 ceph_df
-rw-r--r--. 1 root root    3195 Oct 15 04:11 ceph_fs_dump
-rw-r--r--. 1 root root     211 Oct 15 04:11 ceph_fs_ls
-rw-r--r--. 1 root root     214 Oct 15 04:11 ceph_mds_stat
-rw-r--r--. 1 root root     458 Oct 15 04:11 ceph_mon_dump
-rw-r--r--. 1 root root      17 Oct 15 04:11 ceph_osd_blocked-by
-rw-r--r--. 1 root root    2683 Oct 15 04:11 ceph_osd_df_tree
-rw-r--r--. 1 root root    1535 Oct 15 04:11 ceph_osd_df
-rw-r--r--. 1 root root    4801 Oct 15 04:11 ceph_osd_dump
-rw-r--r--. 1 root root     897 Oct 15 04:11 ceph_osd_numa-status
-rw-r--r--. 1 root root     559 Oct 15 04:11 ceph_osd_perf
-rw-r--r--. 1 root root     960 Oct 15 04:11 ceph_osd_pool_autoscale-status
-rw-r--r--. 1 root root    1297 Oct 15 04:11 ceph_osd_pool_ls_detail
-rw-r--r--. 1 root root      57 Oct 15 04:11 ceph_osd_stat
-rw-r--r--. 1 root root   48518 Oct 15 04:11 ceph_pg_dump
-rw-r--r--. 1 root root      77 Oct 15 04:11 ceph_pg_stat
-rw-r--r--. 1 root root     708 Oct 15 04:11 ceph_status
-rw-r--r--. 1 root root     626 Oct 15 04:11 ceph_time-sync-status
-rw-r--r--. 1 root root   94037 Oct 15 04:12 ceph_config-key_dump
drwx------. 2 root root    4096 Oct 15 04:18 json_output
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# cat ceph_mon/ceph_health_detail 
HEALTH_OK
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# cat ceph_mon/ceph_df
--- RAW STORAGE ---
CLASS     SIZE    AVAIL     USED  RAW USED  %RAW USED
hdd    180 GiB  178 GiB  2.3 GiB   2.3 GiB       1.30
TOTAL  180 GiB  178 GiB  2.3 GiB   2.3 GiB       1.30

--- POOLS ---
POOL                               ID  PGS   STORED  OBJECTS     USED  %USED  MAX AVAIL
.mgr                                1    1  449 KiB        2  1.3 MiB      0     56 GiB
cephfs.cephfs.meta                  2   16   10 KiB       22  118 KiB      0     56 GiB
cephfs.cephfs.data                  3   32      0 B        0      0 B      0     56 GiB
cephfs.cephfs_upgrade_mirror.meta   4   16   22 MiB       31   67 MiB   0.04     56 GiB
cephfs.cephfs_upgrade_mirror.data   5   32  500 MiB      138  1.5 GiB   0.87     56 GiB
[root@ceph1-amk-mirror-9tdhoy-node1-installer sos_commands]# 
jcastill commented 1 month ago

Hi @AmarnatReddy

Most of the ceph_ plugins, if not all, attempt to capture files in /var/log/ceph. It looks like you are checking in the plugin captures inside sos_commands, but the /var/log/ceph dir should be in the parent directory. Can you check if it was written there in your case?

AmarnatReddy commented 1 month ago

Thanks Jose Castillo Logs are getting collected

total 24
drwxr-xr-x.   4 root root   30 Aug 27 07:57 usr
drwxr-xr-x.   5 root root   41 Oct 17 08:49 var
lrwxrwxrwx.   1 root root    7 Oct 17 09:02 lib -> usr/lib
dr-xr-xr-x. 137 root root 4096 Oct 17 09:05 proc
dr-xr-xr-x.  10 root root  112 Oct 17 09:05 sys
dr-xr-xr-x.   5 root root   87 Oct 17 09:05 boot
drwxr-xr-x.  55 root root 4096 Oct 17 10:17 etc
lrwxrwxrwx.   1 root root   29 Oct 17 10:19 mount -> sos_commands/filesys/mount_-l
lrwxrwxrwx.   1 root root   37 Oct 17 10:19 df -> sos_commands/filesys/df_-al_-x_autofs
lrwxrwxrwx.   1 root root   31 Oct 17 10:19 dmidecode -> sos_commands/hardware/dmidecode
lrwxrwxrwx.   1 root root   26 Oct 17 10:19 hostname -> sos_commands/host/hostname
lrwxrwxrwx.   1 root root   24 Oct 17 10:19 uptime -> sos_commands/host/uptime
lrwxrwxrwx.   1 root root   48 Oct 17 10:19 root-symlinks -> sos_commands/host/find_._-maxdepth_2_-type_l_-ls
lrwxrwxrwx.   1 root root   28 Oct 17 10:20 uname -> sos_commands/kernel/uname_-a
lrwxrwxrwx.   1 root root   25 Oct 17 10:20 lsmod -> sos_commands/kernel/lsmod
lrwxrwxrwx.   1 root root   23 Oct 17 10:20 last -> sos_commands/login/last
lrwxrwxrwx.   1 root root   90 Oct 17 10:20 vgdisplay -> sos_commands/lvm2/vgdisplay_-vv_--config_global_metadata_read_only_1_--nolocking_--foreign
lrwxrwxrwx.   1 root root   24 Oct 17 10:20 free -> sos_commands/memory/free
lrwxrwxrwx.   1 root root   34 Oct 17 10:20 ip_addr -> sos_commands/networking/ip_-o_addr
lrwxrwxrwx.   1 root root   47 Oct 17 10:20 ip_route -> sos_commands/networking/ip_route_show_table_all
lrwxrwxrwx.   1 root root   41 Oct 17 10:20 netstat -> sos_commands/networking/netstat_-W_-neopa
lrwxrwxrwx.   1 root root   28 Oct 17 10:20 lspci -> sos_commands/pci/lspci_-nnvv
lrwxrwxrwx.   1 root root   31 Oct 17 10:20 ps -> sos_commands/process/ps_auxwwwm
lrwxrwxrwx.   1 root root   31 Oct 17 10:20 pstree -> sos_commands/process/pstree_-lp
drwxr-xr-x.  10 root root  126 Oct 17 10:20 run
lrwxrwxrwx.   1 root root   80 Oct 17 10:20 installed-rpms -> sos_commands/rpm/sh_-c_rpm_--nodigest_-qa_--qf_-59_NVRA_INSTALLTIME_date_sort_-V
lrwxrwxrwx.   1 root root   32 Oct 17 10:20 date -> sos_commands/systemd/timedatectl
drwxr-xr-x.  78 root root 4096 Oct 17 10:20 sos_commands
lrwxrwxrwx.   1 root root   24 Oct 17 10:20 rpm-Va -> sos_commands/rpm/rpm_-Va
-rw-r--r--.   1 root root  203 Oct 17 10:20 environment
-rw-r--r--.   1 root root   16 Oct 17 10:21 version.txt
drwxr-xr-x.   2 root root   35 Oct 17 10:21 sos_logs
drwxr-xr-x.   2 root root   74 Oct 17 10:21 sos_reports
[root@ceph1-mirror-up-w4tqx4-node1-installer sosreport-ceph1-mirror-up-w4tqx4-node1-installer-test-2024-10-17-skfzcab]# cd /var/
.updated  adm/      cache/    crash/    db/       empty/    ftp/      games/    kerberos/ lib/      local/    lock/     log/      mail/     nis/      opt/      preserve/ run/      spool/    tmp/      yp/       
[root@ceph1-mirror-up-w4tqx4-node1-installer sosreport-ceph1-mirror-up-w4tqx4-node1-installer-test-2024-10-17-skfzcab]# cd /var/lo
local/ lock/  log/   
[root@ceph1-mirror-up-w4tqx4-node1-installer sosreport-ceph1-mirror-up-w4tqx4-node1-installer-test-2024-10-17-skfzcab]# cd /var/log/
README                 ceph/                  cloud-init.log         dnf.log                insights-client/       maillog                qemu-ga/               spooler                tuned/
audit/                 chrony/                cron                   dnf.rpm.log            kdump.log              messages               rhsm/                  sssd/                  wtmp
btmp                   cloud-init-output.log  dnf.librepo.log        hawkey.log             lastlog                private/               secure                 tallylog               
[root@ceph1-mirror-up-w4tqx4-node1-installer sosreport-ceph1-mirror-up-w4tqx4-node1-installer-test-2024-10-17-skfzcab]# cd /var/log/ceph/
5d3c2e12-8c8a-11ef-b799-fa163e147a61/ cephadm.log                           
[root@ceph1-mirror-up-w4tqx4-node1-installer sosreport-ceph1-mirror-up-w4tqx4-node1-installer-test-2024-10-17-skfzcab]# 
AmarnatReddy commented 3 days ago

I have used this command and it is saying the ceph plugin is not supported without that it is not collecting /var/log/ceph logs

[root@ceph1-amk-weekly-kvjk8q-node1-installer tmp]# sudo sosreport -a --all-logs -e ceph --batch
Please note the 'sosreport' command has been deprecated in favor of the new 'sos' command, E.G. 'sos report'.
Redirecting to 'sos report -a --all-logs -e ceph --batch'

sosreport (version 4.7.2)

a non-existing plugin (ceph) was specified in the command line.
[root@ceph1-amk-weekly-kvjk8q-node1-installer tmp]# 

I removed and collected all-logs. In Previous comment I checked in /var/log instead of checking inside the sosreport

[root@ceph1-amk-weekly-kvjk8q-node1-installer tmp]# cd sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl
[root@ceph1-amk-weekly-kvjk8q-node1-installer sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl]# ls -lrt
total 24
drwxr-xr-x.   4 root root   30 Nov 14 16:26 usr
lrwxrwxrwx.   1 root root    7 Nov 14 16:26 lib -> usr/lib
drwxr-xr-x.   5 root root   41 Nov 26 10:12 var
dr-xr-xr-x. 158 root root 4096 Nov 26 10:28 proc
dr-xr-xr-x.  10 root root  112 Nov 26 10:29 sys
dr-xr-xr-x.   5 root root   87 Nov 26 10:29 boot
drwxr-xr-x.  55 root root 4096 Nov 27 12:04 etc
lrwxrwxrwx.   1 root root   29 Nov 27 15:09 mount -> sos_commands/filesys/mount_-l
lrwxrwxrwx.   1 root root   37 Nov 27 15:09 df -> sos_commands/filesys/df_-al_-x_autofs
lrwxrwxrwx.   1 root root   31 Nov 27 15:09 dmidecode -> sos_commands/hardware/dmidecode
lrwxrwxrwx.   1 root root   26 Nov 27 15:09 hostname -> sos_commands/host/hostname
lrwxrwxrwx.   1 root root   24 Nov 27 15:09 uptime -> sos_commands/host/uptime
lrwxrwxrwx.   1 root root   48 Nov 27 15:09 root-symlinks -> sos_commands/host/find_._-maxdepth_2_-type_l_-ls
lrwxrwxrwx.   1 root root   28 Nov 27 15:09 uname -> sos_commands/kernel/uname_-a
lrwxrwxrwx.   1 root root   25 Nov 27 15:09 lsmod -> sos_commands/kernel/lsmod
lrwxrwxrwx.   1 root root   23 Nov 27 15:09 last -> sos_commands/login/last
lrwxrwxrwx.   1 root root   90 Nov 27 15:09 vgdisplay -> sos_commands/lvm2/vgdisplay_-vv_--config_global_metadata_read_only_1_--nolocking_--foreign
lrwxrwxrwx.   1 root root   24 Nov 27 15:09 free -> sos_commands/memory/free
lrwxrwxrwx.   1 root root   34 Nov 27 15:09 ip_addr -> sos_commands/networking/ip_-o_addr
lrwxrwxrwx.   1 root root   47 Nov 27 15:09 ip_route -> sos_commands/networking/ip_route_show_table_all
lrwxrwxrwx.   1 root root   41 Nov 27 15:09 netstat -> sos_commands/networking/netstat_-W_-neopa
lrwxrwxrwx.   1 root root   28 Nov 27 15:09 lspci -> sos_commands/pci/lspci_-nnvv
lrwxrwxrwx.   1 root root   31 Nov 27 15:09 ps -> sos_commands/process/ps_auxwwwm
lrwxrwxrwx.   1 root root   31 Nov 27 15:09 pstree -> sos_commands/process/pstree_-lp
lrwxrwxrwx.   1 root root   80 Nov 27 15:09 installed-rpms -> sos_commands/rpm/sh_-c_rpm_--nodigest_-qa_--qf_-59_NVRA_INSTALLTIME_date_sort_-V
drwxr-xr-x.  10 root root  126 Nov 27 15:09 run
lrwxrwxrwx.   1 root root   24 Nov 27 15:09 rpm-Va -> sos_commands/rpm/rpm_-Va
lrwxrwxrwx.   1 root root   32 Nov 27 15:09 date -> sos_commands/systemd/timedatectl
drwxr-xr-x.  78 root root 4096 Nov 27 15:09 sos_commands
-rw-r--r--.   1 root root  203 Nov 27 15:10 environment
-rw-r--r--.   1 root root   16 Nov 27 15:10 version.txt
drwxr-xr-x.   2 root root   35 Nov 27 15:10 sos_logs
drwxr-xr-x.   2 root root   74 Nov 27 15:10 sos_reports
[root@ceph1-amk-weekly-kvjk8q-node1-installer sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl]# pwd
/var/tmp/sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl
[root@ceph1-amk-weekly-kvjk8q-node1-installer sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl]# cd var/log/
audit/                 cloud-init.log         dnf.librepo.log        dnf.rpm.log            kdump.log              messages               secure                 tuned/                 
cloud-init-output.log  cron                   dnf.log                insights-client/       maillog                rhsm/                  spooler                
[root@ceph1-amk-weekly-kvjk8q-node1-installer sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl]# cd var/log/
[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# ls -lrt
total 34304
-rw-------. 1 root root        0 Nov 14 16:26 maillog
-rw-------. 1 root root        0 Nov 14 16:26 spooler
drwxr-xr-x. 2 root root       23 Nov 26 10:12 tuned
drwxr-xr-x. 2 root root       43 Nov 26 10:12 rhsm
-rw-r-----. 1 root adm      9259 Nov 26 10:29 cloud-init-output.log
-rw-r-----. 1 root root   202451 Nov 26 10:29 cloud-init.log
-rw-------. 1 root root     1366 Nov 26 10:29 kdump.log
drwx------. 2 root root       33 Nov 27 06:45 insights-client
drwx------. 2 root root       42 Nov 27 11:00 audit
-rw-------. 1 root root    26423 Nov 27 15:07 secure
-rw-------. 1 root root    16833 Nov 27 15:09 cron
-rw-r--r--. 1 root root    16986 Nov 27 15:09 dnf.rpm.log
-rw-r--r--. 1 root root    54117 Nov 27 15:09 dnf.librepo.log
-rw-r--r--. 1 root root   219109 Nov 27 15:09 dnf.log
-rw-------. 1 root root 34556086 Nov 27 15:11 messages
[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# 
jcastill commented 3 days ago

@AmarnatReddy you are trying to activate the ceph plugin, but it was divided into the following ones:

ceph_ansible
ceph_common
ceph_iscsi
ceph_mds
ceph_mgr
ceph_mon
ceph_osd
ceph_rgw

Can you try any of these directly, or just run 'sos report' without explicitly mentioning the plugin and see if you get the files?

AmarnatReddy commented 3 days ago

@jcastill I tried with this command. I am not seeing the /var/log/ceph logs getting collected

[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# sosreport -a --all-logs --batch
Please note the 'sosreport' command has been deprecated in favor of the new 'sos' command, E.G. 'sos report'.
Redirecting to 'sos report -a --all-logs --batch'

sosreport (version 4.7.2)

on the server /var/log/ceph contents

[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# ls -lrt /var/log/ceph/
total 1984
drwxrwx---. 2 ceph ceph      29 Nov 26 10:45 5a65c3d8-abe3-11ef-aec0-fa163ee915ac
-rw-r--r--. 1 root root 1980588 Nov 27 16:35 cephadm.log

In sosreport /var/log contents

[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# ls -lrt /var/tmp/sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl/var/log 
total 34304
-rw-------. 1 root root        0 Nov 14 16:26 maillog
-rw-------. 1 root root        0 Nov 14 16:26 spooler
drwxr-xr-x. 2 root root       23 Nov 26 10:12 tuned
drwxr-xr-x. 2 root root       43 Nov 26 10:12 rhsm
-rw-r-----. 1 root adm      9259 Nov 26 10:29 cloud-init-output.log
-rw-r-----. 1 root root   202451 Nov 26 10:29 cloud-init.log
-rw-------. 1 root root     1366 Nov 26 10:29 kdump.log
drwx------. 2 root root       33 Nov 27 06:45 insights-client
drwx------. 2 root root       42 Nov 27 11:00 audit
-rw-------. 1 root root    26423 Nov 27 15:07 secure
-rw-------. 1 root root    16833 Nov 27 15:09 cron
-rw-r--r--. 1 root root    16986 Nov 27 15:09 dnf.rpm.log
-rw-r--r--. 1 root root    54117 Nov 27 15:09 dnf.librepo.log
-rw-r--r--. 1 root root   219109 Nov 27 15:09 dnf.log
-rw-------. 1 root root 34556086 Nov 27 15:11 messages
jcastill commented 3 days ago

That's really strange. What do you get if you run:

grep ceph /var/tmp/sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl/sos_logs/sos.log

AmarnatReddy commented 3 days ago

It's big output. I trimmed for quick reference. Attaching complete log

sos.log

Log Reference:

[root@ceph1-amk-weekly-kvjk8q-node1-installer log]# grep ceph /var/tmp/sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl/sos_logs/sos.log
2024-11-27 15:09:03,682 INFO: [archive:TarFileArchive] initialised empty FileCacheArchive at '/var/tmp/sos.lhl3hc4u/sosreport-ceph1-amk-weekly-kvjk8q-node1-installer-2024-11-27-khiobcl'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/etc/ceph/*keyring*'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/*keyring*'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/*/*keyring*'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/*/*/*keyring*'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/osd'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/mon'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/var/lib/ceph/tmp/*mnt*'
2024-11-27 15:09:03,731 INFO: [plugin:ceph_common] adding forbidden path '/etc/ceph/*bindpass*'
2024-11-27 15:09:03,732 INFO: [plugin:ceph_common] added cmd output 'ceph -v'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/etc/ceph/*keyring*'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/*keyring*'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/*/*keyring*'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/*/*/*keyring*'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/osd'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/mon'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/var/lib/ceph/tmp/*mnt*'
2024-11-27 15:09:03,733 INFO: [plugin:ceph_mds] adding forbidden path '/etc/ceph/*bindpass*'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok cache status'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok client ls'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok config diff'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok config show'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok counter dump'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok counter schema'
2024-11-27 15:09:03,801 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok damage ls'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump loads'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump tree'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump_blocked_ops'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump_historic_ops'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump_historic_ops_by_duration'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump_mempools'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok dump_ops_in_flight'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok get subtrees'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok objecter_requests'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok ops'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok perf dump'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok perf histogram dump'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok perf histogram schema'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok perf schema'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok session ls'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok status'
2024-11-27 15:09:03,802 INFO: [plugin:ceph_mds] added cmd output 'ceph daemon /var/run/ceph/5a65c3d8-abe3-11ef-aec0-fa163ee915ac/ceph-mds.cephfs-spec.ceph1-amk-weekly-kvjk8q-node1-installer.cimfaj.asok version'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/etc/ceph/*keyring*'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/var/lib/ceph/**/*keyring*'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/var/lib/ceph/**/osd*'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/var/lib/ceph/**/mon*'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/var/lib/ceph/**/tmp/*mnt*'
2024-11-27 15:09:04,172 INFO: [plugin:ceph_mgr] adding forbidden path '/etc/ceph/*bindpass*'