search

singer-io / tap-gitlab

GNU Affero General Public License v3.0
15 stars 19 forks source link

tap-gitlab not fetching projects data #15

Closed sapedevops closed 5 years ago

sapedevops commented 5 years ago

I am exploring tap-gitlab to collect gitlab data and during its execution, I found that it is not able to fetch projects data however able to fetch data for all other things like groups, commits, users, branches etc.

Below is execution details and config file.

[root@gitlab gitstats]# python -V Python 3.6.8

[root@gitlab-bigquery gitstats]# pip -V pip 9.0.3 from /usr/lib/python3.6/site-packages (python 3.6)

[root@gitlab]# cat tap-config-staging.json {"api_url": "https://xyz-staging.xyzcloud.net/api/v4", "private_token": "xxxxxxxxxxxxxxxxxxxx", "projects": "test/testproject", "groups": "", "start_date": "2019-01-01T00:00:00Z"}

[root@gitlab gitstats]# tap-gitlab --config tap-config-staging.json INFO Starting sync {"type": "SCHEMA", "stream": "projects", "schema": {"type": "object", "properties": {"approvals_before_merge": {"type": ["null", "integer"]}, "archived": {"type": ["null", "boolean"]}, "avatar_url": {"type": ["null", "string"]}, "builds_enabled": {"type": ["null", "boolean"]}, "container_registry_enabled": {"type": ["null", "boolean"]}, "created_at": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, "creator_id": {"type": ["null", "integer"]}, "default_branch": {"type": ["null", "string"]}, "description": {"type": ["null", "string"]}, "forks_count": {"type": ["null", "integer"]}, "http_url_to_repo": {"type": ["null", "string"]}, "id": {"type": ["null", "integer"]}, "issues_enabled": {"type": ["null", "boolean"]}, "last_activity_at": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, "lfs_enabled": {"type": ["null", "boolean"]}, "merge_requests_enabled": {"type": ["null", "boolean"]}, "name": {"type": ["null", "string"]}, "name_with_namespace": {"type": ["null", "string"]}, "namespace": {"type": "object", "properties": {"id": {"type": ["null", "integer"]}, "kind": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "path": {"type": ["null", "string"]}}}, "only_allow_merge_if_all_discussions_are_resolved": {"type": ["null", "boolean"]}, "only_allow_merge_if_build_succeeds": {"type": ["null", "boolean"]}, "open_issues_count": {"type": ["null", "integer"]}, "owner_id": {"type": ["null", "integer"]}, "path": {"type": ["null", "string"]}, "path_with_namespace": {"type": ["null", "string"]}, "permissions": {"type": "object", "properties": {"group_access": {"anyOf": [{"type": "object", "properties": {"access_level": {"type": ["null", "integer"]}, "notification_level": {"type": ["null", "integer"]}}}, {"type": "null"}]}, "project_access": {"anyOf": [{"type": "object", "properties": {"access_level": {"type": ["null", "integer"]}, "notification_level": {"type": ["null", "integer"]}}}, {"type": "null"}]}}}, "public": {"type": ["null", "boolean"]}, "public_builds": {"type": ["null", "boolean"]}, "request_access_enabled": {"type": ["null", "boolean"]}, "shared_runners_enabled": {"type": ["null", "boolean"]}, "shared_with_groups": {"anyOf": [{"type": "array", "items": {"type": "object", "properties": {"group_id": {"type": "integer"}, "group_name": {"type": "string"}, "group_access_level": {"type": "integer"}}}}, {"type": "null"}]}, "snippets_enabled": {"type": ["null", "boolean"]}, "ssh_url_to_repo": {"type": ["null", "string"]}, "star_count": {"type": ["null", "integer"]}, "tag_list": {"anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]}, "visibility_level": {"type": ["null", "integer"]}, "web_url": {"type": ["null", "string"]}, "wiki_enabled": {"type": ["null", "boolean"]}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "branches", "schema": {"type": "object", "properties": {"project_id": {"type": "integer"}, "name": {"type": "string"}, "merged": {"type": ["null", "boolean"]}, "protected": {"type": ["null", "boolean"]}, "developers_can_push": {"type": ["null", "boolean"]}, "developers_can_merge": {"type": ["null", "boolean"]}, "commit_id": {"type": ["null", "string"]}}}, "key_properties": ["project_id", "name"]} {"type": "SCHEMA", "stream": "commits", "schema": {"type": "object", "properties": {"id": {"type": "string"}, "project_id": {"type": "integer"}, "short_id": {"type": "string"}, "title": {"type": ["null", "string"]}, "author_name": {"type": ["null", "string"]}, "author_email": {"type": ["null", "string"]}, "committer_name": {"type": ["null", "string"]}, "committer_email": {"type": ["null", "string"]}, "created_at": {"type": "string", "format": "date-time"}, "message": {"type": ["null", "string"]}, "allow_failure": {"type": ["null", "boolean"]}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "issues", "schema": {"type": "object", "properties": {"project_id": {"type": "integer"}, "milestone_id": {"type": ["null", "integer"]}, "author_id": {"type": ["null", "integer"]}, "assignee_id": {"type": ["null", "integer"]}, "description": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "id": {"type": "integer"}, "iid": {"type": "integer"}, "labels": {"type": "array", "items": {"type": "string"}}, "title": {"type": ["null", "string"]}, "updated_at": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, "created_at": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, "subscribed": {"type": ["null", "boolean"]}, "user_notes_count": {"type": ["null", "integer"]}, "due_date": {"type": ["null", "string"]}, "web_url": {"type": ["null", "string"]}, "confidential": {"type": ["null", "boolean"]}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "project_milestones", "schema": {"type": "object", "properties": {"id": {"type": ["null", "integer"]}, "iid": {"type": ["null", "integer"]}, "project_id": {"type": ["null", "integer"]}, "group_id": {"type": ["null", "integer"]}, "title": {"type": ["null", "string"]}, "description": {"type": ["null", "string"]}, "due_date": {"type": ["null", "string"]}, "start_date": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "updated_at": {"type": "string", "format": "date-time"}, "created_at": {"type": "string", "format": "date-time"}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "group_milestones", "schema": {"type": "object", "properties": {"id": {"type": ["null", "integer"]}, "iid": {"type": ["null", "integer"]}, "project_id": {"type": ["null", "integer"]}, "group_id": {"type": ["null", "integer"]}, "title": {"type": ["null", "string"]}, "description": {"type": ["null", "string"]}, "due_date": {"type": ["null", "string"]}, "start_date": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "updated_at": {"type": "string", "format": "date-time"}, "created_at": {"type": "string", "format": "date-time"}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "users", "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "username": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "avatar_url": {"type": ["null", "string"]}, "web_url": {"type": ["null", "string"]}}}, "key_properties": ["id"]} {"type": "SCHEMA", "stream": "groups", "schema": {"type": "object", "properties": {"id": {"type": ["null", "integer"]}, "name": {"type": ["null", "string"]}, "path": {"type": ["null", "string"]}, "description": {"type": ["null", "string"]}, "visibility_level": {"type": ["null", "integer"]}, "lfs_enabled": {"type": ["null", "boolean"]}, "avatar_url": {"type": ["null", "string"]}, "web_url": {"type": ["null", "string"]}, "request_access_enabled": {"type": ["null", "boolean"]}, "full_name": {"type": ["null", "string"]}, "full_path": {"type": ["null", "string"]}, "projects": {"anyOf": [{"type": "array", "items": {"type": "object", "properties": {"id": {"type": "integer"}}}}, {"type": "null"}]}}}, "key_properties": ["id"]} INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/test%2Ftestproject?private_token=xxxxxxxxxxxxxxxxxxxx WARNING Removed 20 paths during transforms: _links ci_config_path ci_default_git_depth empty_repo external_authorization_classification_label import_error import_status jobs_enabled merge_method namespace.avatar_url namespace.full_path namespace.parent_id namespace.web_url only_allow_merge_if_pipeline_succeeds printing_merge_request_link_enabled public_jobs readme_url resolve_outdated_diff_discussions runners_token visibility WARNING Removed paths list: ['_links', 'ci_config_path', 'ci_default_git_depth', 'empty_repo', 'external_authorization_classification_label', 'import_error', 'import_status', 'jobs_enabled', 'merge_method', 'namespace.avatar_url', 'namespace.full_path', 'namespace.parent_id', 'namespace.web_url', 'only_allow_merge_if_pipeline_succeeds', 'printing_merge_request_link_enabled', 'public_jobs', 'readme_url', 'resolve_outdated_diff_discussions', 'runners_token', 'visibility'] INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/3/repository/branches?page=1&private_token=xxxxxxxxxxxxxxxxxxxx {"type": "RECORD", "stream": "branches", "record": {"name": "master", "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "project_id": 3, "commit_id": "5aac4db7bdd9405206a36ea387f2c0db61bfe826"}, "time_extracted": "2019-08-29T11:14:27.613902Z"} WARNING Removed 2 paths during transforms: can_push default WARNING Removed paths list: ['can_push', 'default'] INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/3/repository/commits?page=1&private_token=xxxxxxxxxxxxxxxxxxxx {"type": "RECORD", "stream": "commits", "record": {"id": "5aac4db7bdd9405206a36ea387f2c0db61bfe826", "short_id": "5aac4db7", "created_at": "2019-08-23T13:34:47.000000Z", "title": "Initial commit", "message": "Initial commit", "author_name": "Rohit Varshney", "author_email": "rohvarsh@xyz.net", "committer_name": "Rohit Varshney", "committer_email": "rohvarsh@xyz.net", "project_id": 3}, "time_extracted": "2019-08-29T11:14:27.644121Z"} WARNING Removed 3 paths during transforms: authored_date committed_date parent_ids WARNING Removed paths list: ['authored_date', 'committed_date', 'parent_ids'] INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/3/issues?page=1&private_token=xxxxxxxxxxxxxxxxxxxx INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/3/milestones?page=1&private_token=xxxxxxxxxxxxxxxxxxxx INFO GET https://xyz-staging.xyzcloud.net/api/v4/projects/3/users?page=1&private_token=xxxxxxxxxxxxxxxxxxxx {"type": "RECORD", "stream": "users", "record": {"id": 2, "name": "Rohit Varshney", "username": "rohvarsh", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/31a3edfcf80c508c32fb2e57273c6cfc?s=80&d=identicon", "web_url": "https://xyz-staging.xyzcloud.net/rohvarsh"}, "time_extracted": "2019-08-29T11:14:27.726471Z"} {"type": "RECORD", "stream": "projects", "record": {"id": 3, "description": "", "name": "testproject", "name_with_namespace": "test / testproject", "path": "testproject", "path_with_namespace": "test/testproject", "created_at": "2019-08-23T13:34:47.439000Z", "default_branch": "master", "tag_list": [], "ssh_url_to_repo": "git@xyz-staging.xyzcloud.net:test/testproject.git", "http_url_to_repo": "https://xyz-staging.xyzcloud.net/test/testproject.git", "web_url": "https://xyz-staging.xyzcloud.net/test/testproject", "avatar_url": null, "star_count": 0, "forks_count": 0, "last_activity_at": "2019-08-23T13:34:47.439000Z", "namespace": {"id": 3, "name": "test", "path": "test", "kind": "group"}, "archived": false, "container_registry_enabled": true, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "snippets_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 2, "open_issues_count": 0, "shared_with_groups": [], "request_access_enabled": false, "only_allow_merge_if_all_discussions_are_resolved": false, "permissions": {"project_access": null, "group_access": null}, "owner_id": null, "users": [2]}, "time_extracted": "2019-08-29T11:14:27.557934Z"} {"type": "STATE", "value": {"project_3": "2019-08-23T13:34:47.439000Z"}} INFO Sync complete

sapedevops commented 5 years ago

False issue.