Closed GrayStranger closed 1 year ago
setup.cfg
[general]
short_name = SHRTNM
update = true
min_update_delay = 60
debug = false
logs_dir = /home/grimoire/logs
bulk_size = 1000
scroll_size = 100
aliases_file = /home/grimoire/conf/aliases.json
[projects]
projects_file = /home/grimoire/conf/projects.json
[es_collection]
url = http://elasticsearch:9200
[es_enrichment]
url = http://elasticsearch:9200
autorefresh = true
[sortinghat]
host = mariadb
user = root
password =
database = demo_sh
load_orgs = true
orgs_file = /home/grimoire/conf/organizations.json
autoprofile = [jira, gitlab, git]
matching = [email,email-name,username]
sleep_for = 30
affiliate = true
unaffiliated_group = Unknown
strict_mapping = false
reset_on_load = false
identities_format = grimoirelab
identities_file = [/home/grimoire/conf/identities.yml]
autogender = true
[panels]
kibiter_time_from = now-90d
kibiter_default_index = git
kibiter_url = http://kibiter:5601
kibiter_version = 6.8.6-3
gitlab-merges = true
code-license = true
code-complexity = true
[phases]
collection = true
identities = true
enrichment = true
panels = true
[git]
raw_index = git-raw
enriched_index = git_enriched
latest-items = true
studies = [enrich_demography:git, enrich_areas_of_code:git, enrich_onion:git, enrich_git_branches:git]
[gitlab:merge]
category = merge_request
raw_index = gitlab-mrs_raw
enriched_index = gitlab-mrs_enriched
api-token = ***
sleep-for-rate = true
no-archive = true
studies = [enrich_onion:gitlab-merge]
enterprise-url = "***"
[jira]
raw_index = jira_raw
enriched_index = jira_enrich
no-archive = true
backend-user = ***
backend-password = ***
#[rocketchat]
#raw_index = rocketchat_raw
#enriched_index = rocketchat_enriched
#api-token = ***
#sleep-for-rate = true
#user-id = ***
#no-archive = true
[cocom]
raw_index = cocom_chaoss
enriched_index = cocom_chaoss_enrich
category = code_complexity_lizard_file
studies = [enrich_cocom_analysis]
branches = main
[enrich_cocom_analysis]
out_index = cocom_chaoss_study
interval_months = [3]
[colic]
raw_index = colic_chaoss
enriched_index = colic_chaoss_enrich
category = code_license_nomos
studies = [enrich_colic_analysis]
exec-path = /usr/share/fossology/nomos/agent/nomossa
branches = main
[enrich_colic_analysis]
#[confluence]
#raw_index = confluence_raw
#enriched_index = confluence_enriched
#no-archive = true
[enrich_git_branches:git]
[enrich_demography:git]
[enrich_areas_of_code:git]
in_index = git-raw
out_index = git-aoc_enriched
[enrich_onion:git]
in_index = git_enriched
out_index = git-onion_enriched
[enrich_onion:gitlab-merge]
in_index = gitlab-mrs_enriched
out_index = gitlab-mrs-onion_enriched
data_source = gitlab-merges
[enrich_forecast_activity]
out_index = git_study_forecast
[enrich_reference_analysis]
Any ideas what's going wrong? :)
Hi @FixItNowPlease
Could you tell me your mordred version?
Hi @FixItNowPlease
Could you tell me your mordred version?
Hi!
0.7.1 based on docker-compose conf, I rolled back from latest because of connection to mariadb issue (https://github.com/chaoss/grimoirelab/issues/571)
I just figured out that there is no "git-raw" alias in elastic aliases and this in logs:
grimoire_elk.elastic - WARNING - Something went wrong when adding an alias on http://elasticsearch:9200/git-raw. Alias not set. 400 Client Error: Bad Request for url: http://elasticsearch:9200/_aliases
git section of aliases.json
"git": {
"raw": [
{
"alias": "git-raw"
}
],
"enrich": [
{
"alias": "git"
},
{
"alias": "git_author"
},
{
"alias": "git_enrich"
},
{
"alias": "affiliations"
},
{
"alias": "all_enriched"
}
]
},
and this in "setup.cfg":
[git]
raw_index = git-raw
enriched_index = git_enriched
latest-items = true
studies = [enrich_demography:git, enrich_areas_of_code:git, enrich_onion:git, enrich_git_branches:git]
[enrich_areas_of_code:git]
in_index = git-raw
out_index = git-aoc_enriched
indexes:
green open .kibana_5 GX5p_BC6Tm2hB4sphWDn4w 1 0 222 3 313.3kb 313.3kb
yellow open jira_enrich RnPKFKdWTb-VrfsQcVNdgQ 5 1 404333 113864 1.4gb 1.4gb
green open .kibana_8 5wY-bdidQdmzAVn2QOUy_g 1 0 222 3 313.3kb 313.3kb
yellow open cocom_chaoss 4g9jybfLTh6iDfwqNxJUIQ 5 1 978 157 3.8mb 3.8mb
yellow open gitlab-mrs_enriched od6-orTyRMqTGmzk9ceK2g 5 1 226 15 453.7kb 453.7kb
green open .kibana_2 8b_HIoP-TJWYarHTwN1edA 1 0 222 3 313.3kb 313.3kb
green open .kibana_9 RV17j5TDRVC4BMtDgbaU8A 1 0 222 3 315.7kb 315.7kb
yellow open git-raw uMyAHdwNSoG_6rOvZFKaGg 5 1 5542 646 25.6mb 25.6mb
yellow open cocom_chaoss_enrich 9mMUIgwFTsC0E9Nt6nO2-Q 5 1 0 0 1.2kb 1.2kb
yellow open colic_chaoss_enrich _hGzw1CgRS-tRebFOOpiog 5 1 0 0 1.2kb 1.2kb
green open .kibana_6 vjD1TIkXRMuk1s5-S6TH9g 1 0 222 3 313.3kb 313.3kb
green open .kibana_3 UbJAOb4kTPehI0fVbfiw0w 1 0 222 3 313.3kb 313.3kb
green open .kibana_10 w5-cjllyROeBOGxbHO0zqw 1 0 222 3 313.3kb 313.3kb
yellow open git_enriched PRshHuHWSbyguSCpg9dC0Q 5 1 5542 1589 16.7mb 16.7mb
yellow open gitlab-mrs_raw 3eHyDVe0QliMMPP-1l7W1w 5 1 226 12 4mb 4mb
green open .kibana_7 pse7SRwwTCOesbt1VXgyEA 1 0 222 3 313.3kb 313.3kb
yellow open jira_raw n1Jm6rZLTFCGafFYkllVAg 5 1 88703 0 1.9gb 1.9gb
yellow open .grimoirelab-sigils NMQ0V15CR1SkXbs7-oiShg 5 1 39 2 96.1kb 96.1kb
yellow open colic_chaoss RNWZyqGmQGW2C1EnjSsVCA 5 1 753 21 823.9kb 823.9kb
green open .kibana_1 sgRdH1JhRaOAjmRyozlLQQ 1 0 222 27 384.5kb 384.5kb
green open .kibana_4 nuWVZ8rQQO6zKCbk5zW3Tg 1 0 222 3 313.3kb 313.3kb
aliases:
jira-raw jira_raw - - -
affiliations jira_enrich - - -
all_enriched jira_enrich - - -
all_enriched_tickets jira_enrich - - -
jira jira_enrich - - -
jira_resolution_date jira_enrich - - -
cocom cocom_chaoss_enrich - - -
affiliations gitlab-mrs_enriched - - -
all_enriched gitlab-mrs_enriched - - -
gitlab_merge_requests gitlab-mrs_enriched - - -
colic colic_chaoss_enrich - - -
colic-raw colic_chaoss - - -
cocom-raw cocom_chaoss - - -
affiliations git_enriched - - -
all_enriched git_enriched - - -
git git_enriched - - -
git_author git_enriched - - -
git_enrich git_enriched - - -
gitlab_merge_requests-raw gitlab-mrs_raw - - -
.kibana .kibana_10 - - -
no idea is it connected and how to fix in case if it is.
It seems that Mordred cannot create the alias git-raw
because you named the raw_index
the same as the alias. Could you try it but use another name like git_raw
?
[git]
raw_index = git_raw
enriched_index = git_enriched
latest-items = false
studies = [enrich_demography:git, enrich_areas_of_code:git, enrich_onion:git, enrich_git_branches:git]
[enrich_areas_of_code:git]
in_index = git_raw
out_index = git-aoc_enriched
PS: deactivate latest-items = false
or remove the clones
i had the same thought yesterday evening and already done it, now i have this in indexes and aliases, no corresponding errors in log. But "string indices must be integers" - still in place.
yellow open gitlab-mrs_enriched 3IhJp1LTTPO6QikHPmeKBw 5 1 226 0 407.7kb 407.7kb
yellow open cocom_chaoss_enrich Ctex26kfQXaXyb4_dTvDIg 5 1 0 0 1.2kb 1.2kb
yellow open jira_enriched Ky3pS7hEROuzlRK9EEuA5g 5 1 404333 67951 2.4gb 2.4gb
yellow open .grimoirelab-sigils FT_hBfluQ8ujaCtWx7WD4Q 5 1 39 2 79.7kb 79.7kb
yellow open git_raw MzxbIEqySxyngaNFFj6eXw 5 1 2686 0 5mb 5mb
yellow open cocom_chaoss oLPij7K5TYGp65BjMoy-zA 5 1 712 0 1.7mb 1.7mb
yellow open colic_chaoss qWZizWvQQ3CBNhTIcSbV8g 5 1 615 0 998.9kb 998.9kb
yellow open jira_raw s90XJ9pwTT6n_G3p4hUB3A 5 1 88703 0 1.9gb 1.9gb
yellow open git_enriched G1uMgVmiSFSIuP_-95CmZA 5 1 2686 0 4.8mb 4.8mb
green open .kibana_1 zjug6pspSRaA3DymhGKt-A 1 0 222 27 374.5kb 374.5kb
yellow open colic_chaoss_enrich p6K1gyTPQZqeub5Y4wbpGg 5 1 0 0 1.2kb 1.2kb
yellow open gitlab-mrs_raw QQKCAkYPTr6SXBChv1M6GQ 5 1 226 0 3.5mb 3.5mb
affiliations git_enriched - - -
all_enriched git_enriched - - -
git git_enriched - - -
git_author git_enriched - - -
git_enrich git_enriched - - -
cocom-raw cocom_chaoss - - -
jira-raw jira_raw - - -
git-raw git_raw - - -
colic colic_chaoss_enrich - - -
cocom cocom_chaoss_enrich - - -
.kibana .kibana_1 - - -
affiliations jira_enriched - - -
all_enriched jira_enriched - - -
all_enriched_tickets jira_enriched - - -
jira jira_enriched - - -
jira_resolution_date jira_enriched - - -
affiliations gitlab-mrs_enriched - - -
all_enriched gitlab-mrs_enriched - - -
gitlab_merge_requests gitlab-mrs_enriched - - -
gitlab_merge_requests-raw gitlab-mrs_raw - - -
colic-raw colic_chaoss - - -
and after index name has been corrected - i had restarted all from scratch.
sudo docker compose down --remove-orphans
sudo docker system prune -a --volumes
sudo rm -rf ~/stat.bot/logs/all.log
sudo rm -rf ~/stat.bot/data/elasticsearch/nodes
I cannot reproduce the error
2023-05-10 14:27:58,999 - sirmordred.task_collection - INFO - [git] collection phase starts
2023-05-10 14:27:59,001 - sirmordred.task_collection - INFO - [git] collection starts for https://github.com/chaoss/grimoirelab.git
...
2023-05-10 14:28:36,230 - sirmordred.task_collection - INFO - [git] collection phase finished in 00:00:37
2023-05-10 14:28:46,251 - sirmordred.task_enrich - INFO - [git] enrichment phase starts
...
2023-05-10 14:28:53,501 - sirmordred.task_enrich - INFO - [git] enrichment phase finished in 0:00:07
...
2023-05-10 14:28:59,196 - sirmordred.task_enrich - INFO - [git] studies phase start
2023-05-10 14:29:01,283 - sirmordred.task_enrich - INFO - [git] Executing studies ['enrich_demography:git', 'enrich_areas_of_code:git', 'enrich_onion:git']
2023-05-10 14:29:01,285 - grimoire_elk.elk - INFO - [git] Starting study: enrich_demography:git, params {'alias': 'demographics'}
2023-05-10 14:29:01,285 - grimoire_elk.enriched.enrich - INFO - [git] Demography starting study http://elasticsearch:9200/git_demo_enriched
2023-05-10 14:29:02,545 - grimoire_elk.enriched.enrich - INFO - [git] Demography Creating alias: demographics
2023-05-10 14:29:02,571 - grimoire_elk.elastic - INFO - Alias demographics created on http://elasticsearch:9200/git_demo_enriched.
2023-05-10 14:29:02,571 - grimoire_elk.enriched.enrich - INFO - [git] Demography end http://elasticsearch:9200/git_demo_enriched
2023-05-10 14:29:02,571 - grimoire_elk.elk - INFO - [git] Starting study: enrich_areas_of_code:git, params {'in_index': 'git_demo_raw', 'out_index': 'git-aoc_demo_enriched', 'alias': 'git_areas_of_code'}
...
2023-05-10 14:29:12,204 - grimoire_elk.enriched.git - INFO - [git] study areas_of_code creating alias: git_areas_of_code
2023-05-10 14:29:12,214 - grimoire_elk.enriched.git - INFO - [git] study areas_of_code end
2023-05-10 14:29:12,223 - grimoire_elk.elk - INFO - [git] Starting study: enrich_onion:git, params {'in_index': 'git', 'out_index': 'git-onion_demo_enriched', 'contribs_field': 'hash', 'alias': 'all_onion'}
...
2023-05-10 14:29:19,801 - grimoire_elk.enriched.enrich - INFO - [git] study onion Creating alias: all_onion
2023-05-10 14:29:19,811 - grimoire_elk.enriched.enrich - INFO - [git] study onion end
2023-05-10 14:29:19,812 - sirmordred.task_enrich - INFO - [git] studies phase end
2023-05-10 14:29:19,812 - sirmordred.task_enrich - INFO - [git] autorefresh for studies start
2023-05-10 14:29:19,877 - sirmordred.task_enrich - INFO - [git] Refreshing identities
2023-05-10 14:29:30,005 - sirmordred.task_enrich - INFO - [git] autorefresh for studies end
2023-05-10 14:29:30,006 - sirmordred.task_manager - INFO - [git] sleeping for 60 seconds
Here is my setup.cfg
[git]
raw_index = git_demo_raw
enriched_index = git_demo_enriched
latest-items = true
studies = [enrich_demography:git, enrich_areas_of_code:git, enrich_onion:git]
[enrich_demography:git]
[enrich_areas_of_code:git]
in_index = git_demo_raw
out_index = git-aoc_demo_enriched
[enrich_onion:git]
in_index = git
out_index = git-onion_demo_enriched
contribs_field = hash
Indexes:
GET _cat/indices?s=i
yellow open git-aoc_demo_enriched Iv2j3INsS6KwQTRnrlel_A 5 1 3110 0 3.2mb 3.2mb
yellow open git-onion_demo_enriched Pw4B80d-QLegUBxHLh0roA 5 1 464 0 397.9kb 397.9kb
yellow open git_demo_enriched O2wMSW_aRCKbWG-eaLjPPA 5 1 643 5 1.8mb 1.8mb
yellow open git_demo_raw MDk1I0hATuiESqXB4G4yBQ 5 1 643 0 1.4mb 1.4mb
I'm using these versions
docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6
bitergia/kibiter:community-v6.8.6-3
grimoirelab/grimoirelab:0.7.1
mariadb:10.6
$ docker ps | awk '{print $2}'
ID
grimoirelab/grimoirelab:0.7.1
bitergia/kibiter:community-v6.8.6-3
docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6
mariadb:10.6
Could you run these command?
$ docker exec <mordred_container_name> tail -n 21 aliases.json
"studies_aliases": {
"enrich": [
{
"alias": "all_onion",
"name": "enrich_onion"
},
{
"alias": "demographics",
"name": "enrich_demography"
},
{
"alias": "demographics_contribution",
"name": "enrich_demography_contribution"
},
{
"alias": "git_areas_of_code",
"name": "enrich_areas_of_code"
}
]
}
}
$ docker exec <mordred_container_name> tail -n 21 conf/aliases.json
tail: cannot open 'con/aliases.json' for reading: No such file or directory
I see in your setup.cfg that your aliases file are in /home/grimoire/conf/aliases.json
[general]
aliases_file = /home/grimoire/conf/aliases.json
I see the difference between my one and yours setup.cfg:
your setup:
[enrich_onion:git]
in_index = git
out_index = git-onion_demo_enriched
**contribs_field = hash**
my one:
[enrich_onion:git]
in_index = git_enriched
out_index = git-onion_enriched
My versions of images:
ID
grimoirelab/hatstall
docker-compose-mordred
bitergia/kibiter:community-v6.8.6-3
mariadb:10.9.3
docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.18
containrrr/watchtower
it seems that im not needed in watchtower because of stricketed versions of images but...
im rebuilding mordred image on load: docker-compose:
mordred:
restart: on-failure:100
build:
context: .
dockerfile: mordred.upgraded.yml
volumes:
- ../grimoirelab-settings/setup.cfg:/home/grimoire/conf/setup.cfg
- ../grimoirelab-settings/aliases.json:/home/grimoire/conf/aliases.json
- ../grimoirelab-settings/projects.json:/home/grimoire/conf/projects.json
- ../grimoirelab-settings/organizations.json:/home/grimoire/conf/organizations.json
- ../grimoirelab-settings/identities.yml:/home/grimoire/conf/identities.yml
- ../logs:/home/grimoire/logs
depends_on:
- mariadb
- elasticsearch
mem_limit: 4g
dockerfile mordred.upgraded.yml:
FROM grimoirelab/grimoirelab:0.7.1
RUN sudo apt update
RUN sudo apt install -y cloc && sudo apt install -y wget
RUN wget https://github.com/fossology/fossology/releases/download/3.11.0/FOSSology-3.11.0-ubuntu-focal.tar.gz
RUN tar -xzf FOSSology-3.11.0-ubuntu-focal.tar.gz
RUN sudo apt-get -y install ./packages/fossology-common_3.11.0-1_amd64.deb && wget http://security.ubuntu.com/ubuntu/pool/main/j/json-c/libjson-c4_0.13.1+dfsg-7ubuntu0.3_amd64.deb
RUN sudo apt-get -y install ./libjson-c4_0.13.1+dfsg-7ubuntu0.3_amd64.deb
RUN sudo apt-get -y install ./packages/fossology-nomos_3.11.0-1_amd64.deb%
Could you run these command?
you want me to add aliases for "studies_aliases" in aliases.json in runtime? I could just add it in the file via vi, is it ok?
I see in your setup.cfg that your aliases file are in
/home/grimoire/conf/aliases.json
Yes, all confs there:
grimoire@8ecb6266d80d:~$ ls /home/grimoire/conf/
aliases.json identities.yml organizations.json projects.json requirements.txt setup.cfg
grimoire@8ecb6266d80d:~$
I have added aliases in the aliases.json and
contribs_field = hash
to [enrich_onion:git]
and restarted all set of containers (think that only mordred has to be restarted)
so i have got studies working! lets give them some time to generate logs.
You can try to change the aliases file path to aliases_file = /home/grimoire/aliases.json
on your setup.cfg
and remove it from the Mordred volumes - ../grimoirelab-settings/aliases.json:/home/grimoire/conf/aliases.json
. By default, Mordred has that file on /home/grimoire/aliases.json
it seems that i lost "affiliations/identities" for git :(... all identities separated again. It is still OK for Jira and at the begining point for git... But studies works well - a lot of new erniched data. "Organizations" also in place.
May be it has been overwrited be some study enrichment and has not done yet?
[git] Refreshing identities - present, it seems works, but identities are separated for git.
Update: identities merged (through git with no Jira identity), BUT Profile name == email_name and email field of profile lost (empty)....
This happends because of: contribs_field = hash?
One more question (it's mentioned in first message) related to indexes: errors in visualisations cocom and colic. It seems that it is because of cocom and colic enriched indexes are not fulfilled. Which way i should digging?
released setup.cfg:
[cocom]
raw_index = cocom_chaoss
enriched_index = cocom_chaoss_enrich
category = code_complexity_lizard_file
studies = [enrich_cocom_analysis]
branches = main
[enrich_cocom_analysis]
out_index = cocom_chaoss_study
interval_months = [3]
[colic]
raw_index = colic_chaoss
enriched_index = colic_chaoss_enrich
category = code_license_nomos
studies = [enrich_colic_analysis]
exec-path = /usr/share/fossology/nomos/agent/nomossa
branches = main
[enrich_colic_analysis]
May be it has been overwrited be some study enrichment and has not done yet?
I don't think so, the studies do not separate any identities
[git] Refreshing identities - present, it seems works, but identities are separated for git.
This is strange, do you mean that Mordred in the first [git] Refreshing identities
works but after the studies, these are separated for git?
According to your configuration (matching = [email,email-name,username]
), it will merge if the name, email, or username are the same.
Update: identities merged (through git with no Jira identity), BUT Profile name == email_name and email field of profile lost (empty)....
I'm not sure but maybe some users fill their profile and set the name with the email.
This happends because of: contribs_field = hash?
No, by default Mordred will always add that field if it is not set.
Error in visualization
Request to Elasticsearch failed: {"error":{"root_cause":[{"type":"query_shard_exception","reason":"No mapping found for [grimoire_creation_date] in order to sort on","index_uuid":"_hGzw1CgRS-tRebFOOpiog","index":"colic_chaoss_enrich"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":"colic_chaoss_enrich","node":"S80PmMojRfqPblvaEGD_1w","reason":{"type":"query_shard_exception","reason":"No mapping found for [grimoire_creation_date] in order to sort on","index_uuid":"_hGzw1CgRS-tRebFOOpiog","index":"colic_chaoss_enrich"}}]},"status":400}
All enriched indexes should have the grimoire_creation_date
field, could you remove your old enriched and studies indexes of cocom
and colic
and let Mordred create them again?
You can go to Discover
, select the index pattern (cocom or colic), and check if they have grimoire_creation_date
field
It seems that cocom and colic indexes had not been enriched for some reason:
yellow open jira_raw 9VuIrFcVSzO8RDVu1aZPZA 5 1 88710 0 1.9gb 1.9gb
green open .kibana_1 dBaqejz5S_Cdh7K9HERMEg 1 0 222 30 396kb 396kb
yellow open cocom_chaoss_enrich Py9ZOCeQT1CyADod3Tu_UA 5 1 0 0 1.2kb 1.2kb
yellow open git-onion_enriched uMZ0ghs_TtKav9q20tc6bg 5 1 424 0 276.5kb 276.5kb
yellow open colic_enrich_graal_repo QLhAZkqDSwulYDdV6tz-cw 5 1 0 0 1.2kb 1.2kb
yellow open gitlab-mrs_enriched SPSM7OgCTjy5rR3dSKEgTg 5 1 226 16 761.6kb 761.6kb
yellow open gitlab-mrs_raw LoHikloiQGmle7mZ39yPwA 5 1 226 12 4mb 4mb
yellow open cocom_chaoss UNonOAOaShW_KqvNPzVgpg 5 1 433 92 1.5mb 1.5mb
yellow open colic_chaoss_enrich Z_rUvr8STz2ZBVDnzG4Wnw 5 1 0 0 1.2kb 1.2kb
yellow open gitlab-mrs-onion_enriched D543vaNGSBa5-EeqrHd8hQ 5 1 176 0 166.7kb 166.7kb
yellow open cocom_chaoss_study dakAZDiLRNap2SULFGKmkg 5 1 0 0 1.2kb 1.2kb
yellow open git_enriched bHUu-2T0SByldj6qE2ptuA 5 1 2686 743 16.3mb 16.3mb
yellow open .grimoirelab-sigils n7LjlMTqSwWEgflr3amTzA 5 1 39 2 79.7kb 79.7kb
yellow open git-aoc_enriched e9lPQXH0SWqpdOKH99AfxA 5 1 22736 946 23.1mb 23.1mb
yellow open colic_chaoss 5i1TXZdYS4Sdp3lNmHOPOA 5 1 709 74 1.1mb 1.1mb
yellow open jira_enriched 7y0OlYGwS0CuAj2arpTysg 5 1 404354 111126 3.4gb 3.4gb
yellow open git_raw aFCLUpzWTZSHj7D0e5RA9g 5 1 2686 0 5mb 5mb
setup.cfg for cocom and colic:
[cocom]
raw_index = cocom_chaoss
enriched_index = cocom_chaoss_enrich
category = code_complexity_lizard_file
studies = [enrich_cocom_analysis]
branches = main
[enrich_cocom_analysis]
out_index = cocom_chaoss_study
interval_months = [3]
[colic]
raw_index = colic_chaoss
enriched_index = colic_chaoss_enrich
category = code_license_nomos
studies = [enrich_colic_analysis]
exec-path = /usr/share/fossology/nomos/agent/nomossa
branches = main
[enrich_colic_analysis]
do I need to specify "branches = main" here? Most of repos has unpredictable names and count of brunches...
You can go to
Discover
, select the index pattern (cocom or colic), and check if they havegrimoire_creation_date
field
no any fields in cocom or colic patterns.
All enriched indexes should have the
grimoire_creation_date
field, could you remove your old enriched and studies indexes ofcocom
andcolic
and let Mordred create them again?
of course, but i don't know how to remove particular index :)
of course, but i don't know how to remove particular index :)
DELETE <index_name>
In your case:
DELETE cocom_chaoss_enrich
DELETE cocom_chaoss_study
DELETE colic_chaoss_enrich
indexes re-created, I have 2 types of errors in log:
2023-05-12 12:33:15,514 - grimoire_elk.elk - ERROR - Error feeding raw from cocom (https://repourl/reponame1.git): git command - 'git <command> [<revision>...] -- [<file>...]'
(return code: 128)
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/elk.py", line 203, in feed_backend
ocean_backend.feed(**params)
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/raw/elastic.py", line 234, in feed
self.feed_items(items)
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/raw/elastic.py", line 250, in feed_items
for item in items:
File "/usr/local/lib/python3.8/site-packages/perceval/backend.py", line 316, in fetch
for item in self.fetch_items(category, **kwargs):
File "/usr/local/lib/python3.8/site-packages/graal/graal.py", line 184, in fetch_items
for commit in commits:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 164, in fetch_items
for commit in commits:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 259, in parse_git_log_from_iter
for commit in parser.parse():
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 542, in parse
for line in self.stream:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 1057, in log
for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 1287, in _exec_nb
raise RepositoryError(cause=cause)
perceval.errors.RepositoryError: git command - 'git <command> [<revision>...] -- [<file>...]'
(return code: 128)
2023-05-12 19:07:43,785 - grimoire_elk.elk - ERROR - Error feeding raw from colic (https://repourl/reponame2): git command - 'git <command> [<revision>...] -- [<file>...]'
(return code: 128)
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/elk.py", line 203, in feed_backend
ocean_backend.feed(**params)
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/raw/elastic.py", line 234, in feed
self.feed_items(items)
File "/usr/local/lib/python3.8/site-packages/grimoire_elk/raw/elastic.py", line 250, in feed_items
for item in items:
File "/usr/local/lib/python3.8/site-packages/perceval/backend.py", line 316, in fetch
for item in self.fetch_items(category, **kwargs):
File "/usr/local/lib/python3.8/site-packages/graal/graal.py", line 184, in fetch_items
for commit in commits:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 164, in fetch_items
for commit in commits:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 259, in parse_git_log_from_iter
for commit in parser.parse():
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 542, in parse
for line in self.stream:
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 1057, in log
for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):
File "/usr/local/lib/python3.8/site-packages/perceval/backends/core/git.py", line 1287, in _exec_nb
raise RepositoryError(cause=cause)
perceval.errors.RepositoryError: git command - 'git <command> [<revision>...] -- [<file>...]'
(return code: 128)
Hi @FixItNowPlease
If the TypeError: string indices must be integers
was fixed using the latest version of the aliases.json (on mordred
container at /home/grimoire/aliases.json
).
[general]
aliases_file = /home/grimoire/aliases.json
Could you please create a new issue with the perceval.errors.RepositoryError: git command - 'git <command> [<revision>...] -- [<file>...]'
error and close this one? This way if someone has the same error it is easier to find it, thanks
Thank you @zhquan for your help. Yes, aliases from container work well. Ok, will open new issue for new error :).
Thanks a lot again.
Hello, all
Thanks in advance for support :)
I have pack of errors in all.log:
At the same time in panel following errors of visualisations (may be it could be connected, some problem with indexes? No idea hoe to fix :(
Indexes:
Aliases: