r10k:deprecation fails with `Faraday::BadRequestError: the server responded with status 400`

[bastelfreak]

running the r10k:deprecation rake task I get:

$ bundle exec rake r10k:deprecation --trace
** Invoke r10k:deprecation (first_time)
** Execute r10k:deprecation
rake aborted!
Faraday::BadRequestError: the server responded with status 400
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/response/raise_error.rb:16:in `on_complete'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/middleware.rb:19:in `block in call'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/response.rb:61:in `on_complete'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/middleware.rb:18:in `call'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday_middleware-1.2.0/lib/faraday_middleware/response_middleware.rb:36:in `call'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday_middleware-1.2.0/lib/faraday_middleware/response/follow_redirects.rb:79:in `perform_with_redirection'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday_middleware-1.2.0/lib/faraday_middleware/response/follow_redirects.rb:67:in `call'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/rack_builder.rb:154:in `build_response'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/connection.rb:516:in `run_request'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/faraday-1.10.0/lib/faraday/connection.rb:200:in `get'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/puppet_forge-3.2.0/lib/puppet_forge/v3/base.rb:69:in `request'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/puppet_forge-3.2.0/lib/puppet_forge/v3/base.rb:76:in `find_request'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/puppet_forge-3.2.0/lib/puppet_forge/v3/base.rb:82:in `find'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bundler/gems/ra10ke-07fbd735ba15/lib/ra10ke/deprecation.rb:47:in `block in deprecated_modules'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bundler/gems/ra10ke-07fbd735ba15/lib/ra10ke/deprecation.rb:43:in `map'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bundler/gems/ra10ke-07fbd735ba15/lib/ra10ke/deprecation.rb:43:in `deprecated_modules'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bundler/gems/ra10ke-07fbd735ba15/lib/ra10ke/deprecation.rb:64:in `bad_mods?'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bundler/gems/ra10ke-07fbd735ba15/lib/ra10ke/deprecation.rb:16:in `block in define_task_deprecation'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:281:in `block in execute'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:281:in `each'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:281:in `execute'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:219:in `block in invoke_with_call_chain'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:199:in `synchronize'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:199:in `invoke_with_call_chain'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/task.rb:188:in `invoke'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:160:in `invoke_task'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:116:in `block (2 levels) in top_level'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:116:in `each'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:116:in `block in top_level'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:125:in `run_with_threads'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:110:in `top_level'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:83:in `block in run'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:186:in `standard_exception_handling'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/lib/rake/application.rb:80:in `run'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/gems/rake-13.0.6/exe/rake:27:in `<top (required)>'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bin/rake:25:in `load'
/home/bastelfreak/code/controlrepo/.vendor/ruby/3.0.0/bin/rake:25:in `<top (required)>'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli/exec.rb:58:in `load'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli/exec.rb:58:in `kernel_load'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli/exec.rb:23:in `run'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli.rb:483:in `exec'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/vendor/thor/lib/thor/invocation.rb:127:in `invoke_command'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/vendor/thor/lib/thor.rb:392:in `dispatch'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli.rb:31:in `dispatch'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/vendor/thor/lib/thor/base.rb:485:in `start'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/cli.rb:25:in `start'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/exe/bundle:48:in `block in <top (required)>'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/lib/bundler/friendly_errors.rb:103:in `with_friendly_errors'
/usr/lib/ruby/gems/3.0.0/gems/bundler-2.3.10/exe/bundle:36:in `<top (required)>'
/usr/bin/bundle:25:in `load'
/usr/bin/bundle:25:in `<main>'
Tasks: TOP => r10k:deprecation

This is from the master branch. I get the same error with the last release. The used Puppetfile:

# used in profiles
mod 'puppet/systemd', :latest
mod 'puppet/lldpd', :latest
mod 'puppet/ferm', :latest
mod 'puppet/borg', :latest
mod 'puppet/wireguard', :latest
mod 'puppet/bird', :latest
mod 'herculesteam/augeasproviders_pam', :latest
mod 'herculesteam/augeasproviders_shellvar', :latest
mod 'puppetlabs/vcsrepo', :latest
mod 'saz/ssh', :latest
mod 'puppet/r10k', :latest
mod 'puppet/dbbackup', :latest
mod 'puppet/mosquitto', :latest
mod 'puppet/grafana', :latest
mod 'puppet/nginx', :latest
mod 'puppet/ssh_keygen', :latest
mod 'puppet/unbound', :latest
mod 'puppetlabs/inifile', :latest
mod 'puppet/prometheus', :latest
mod 'theforeman/foreman', :latest
mod 'theforeman/puppet', :latest
mod 'theforeman/foreman_proxy', :latest
mod 'puppet/nftables', :latest
mod 'herculesteam/augeasproviders_sysctl', :latest
mod 'camptocamp/catalog_diff', :latest
mod 'puppet/unattended_upgrades', :latest
mod 'puppet/selinux', :latest
mod 'choria/choria', :latest
mod 'puppet/archive', :latest
mod 'puppet/elasticsearch', :latest
mod 'jsok/vault',
  git: 'https://github.com/bastelfreak/puppet-vault',
  ref: 'test'

# dependencies
mod 'herculesteam/augeasproviders_core', :latest
mod 'puppetlabs/stdlib', :latest
mod 'choria/mcollective', :latest
mod 'puppetlabs/concat', :latest
mod 'puppetlabs/apt', :latest
mod 'puppetlabs/apache', :latest
mod 'puppetlabs/postgresql', :latest
mod 'puppet/extlib', :latest
mod 'puppet/redis', :latest
mod 'puppet/epel', :latest
mod 'theforeman/puppetserver_foreman', :latest
mod 'richardc/datacat', :latest
mod 'theforeman/dns', :latest
mod 'theforeman/dhcp', :latest
mod 'theforeman/tftp', :latest
mod 'puppetlabs/xinetd', :latest
mod 'choria-mcollective_choria', :latest
mod 'choria/mcollective_agent_puppet', :latest
mod 'choria/mcollective_agent_package', :latest
mod 'choria/mcollective_agent_service', :latest
mod 'choria/mcollective_agent_filemgr', :latest
mod 'choria/mcollective_agent_shell', :latest
mod 'choria/mcollective_agent_nettest', :latest
mod 'choria/mcollective_agent_puppetca', :latest
mod 'choria/mcollective_agent_bolt_tasks', :latest
mod 'choria/mcollective_agent_iptables', :latest
mod 'choria/mcollective_agent_process', :latest
mod 'choria/mcollective_util_actionpolicy', :latest
mod 'optiz0r/mcollective_agent_puppet_env', :latest
mod 'jay7x/mcollective_agent_query', :latest
mod 'choria/mcollective_data_sysctl', :latest
mod 'puppet/yum', :latest
mod 'puppetlabs/java', :latest
mod 'puppet/elastic_stack', :latest
mod 'puppet/hashi_stack', :latest

# core modules are not vendored on Gentoo
mod 'puppetlabs-sshkeys_core', :latest
mod 'puppetlabs-mount_core', :latest
mod 'puppetlabs-augeas_core', :latest
mod 'puppetlabs-selinux_core', :latest
mod 'puppetlabs-yumrepo_core', :latest

[bastelfreak]

@logicminds do you have any ideas here?

[bastelfreak]

The issue comes from: https://github.com/puppetlabs/forge-ruby/blob/51b76eb7eed48be63ca8f03ba717b8527bee8260/lib/puppet_forge/v3/base.rb#L69

[logicminds]

I'll take a look when I get a chance.

[bastelfreak]

ahhhhhhh software.. so the Puppetfile starts with # used in profiles https://github.com/voxpupuli/ra10ke/blob/07fbd735ba156676f3af5b2b15354876aa2989a6/lib/ra10ke/deprecation.rb#L43-L55

forge_modules(puppetfile) seems to return an iterable and the following first hash:

{:namespace=>nil, :name=>nil, :args=>{}}

Since that's total crap, module_name contains only - and the forge URI is v3/modules/- which leads to the HTTP 400 error

[bastelfreak]

and I'm quite sure comments are valid in a Puppetfile, so the parser is broken

[logicminds]

AFAIK, comments are just ignored. So should not be an issue. https://github.com/voxpupuli/ra10ke/blob/master/lib/ra10ke/puppetfile_parser.rb#L33

[bastelfreak]

I'm not sure if that's the correct line. I think https://github.com/voxpupuli/ra10ke/blob/07fbd735ba156676f3af5b2b15354876aa2989a6/lib/ra10ke/puppetfile_parser.rb#L31 should match. https://github.com/voxpupuli/ra10ke/blob/07fbd735ba156676f3af5b2b15354876aa2989a6/lib/ra10ke/monkey_patches.rb#L23-L27

(but looks like it doesn't)

[logicminds]

https://github.com/voxpupuli/ra10ke/pull/new/deprecation-issue

There is a test on this branch for reproduction.

bundle exec rspec spec/ra10ke/deprecation_spec.rb:43

[bastelfreak]

mhm doing it by hand works also :thinking:

irb(main):004:0> file = File.read('Puppetfile')
=> "# used in profiles\nmod 'puppet/systemd', :latest\nmod 'puppet/lldpd', :latest\nmod 'puppet/ferm', :latest\nmod 'puppet/borg', :latest\nmod 'puppet/wireguard', :latest\nmod 'puppet/bird', :latest\nmod 'he...
irb(main):005:0> file.lines.reject { |line| line.match(/#.*\n/) }.join("\n")
=> "mod 'puppet/systemd', :latest\n\nmod 'puppet/lldpd', :latest\n\nmod 'puppet/ferm', :latest\n\nmod 'puppet/borg', :latest\n\nmod 'puppet/wireguard', :latest\n\nmod 'puppet/bird', :latest\n\nmod 'herculesteam/augeasproviders_pam', :latest\n\nmod 'herculesteam/augeasproviders_shellvar', :latest\n\nmod 'puppetlabs/vcsrepo', :latest\n\nmod 'saz/ssh', :latest\n\nmod 'puppet/r10k', :latest\n\nmod 'puppet/dbbackup', :latest\n\nmod 'puppet/mosquitto', :latest\n\nmod 'puppet/grafana', :latest\n\nmod 'puppet/nginx', :latest\n\nmod 'puppet/ssh_keygen', :latest\n\nmod 'puppet/unbound', :latest\n\nmod 'puppetlabs/inifile', :latest\n\nmod 'puppet/prometheus', :latest\n\nmod 'theforeman/foreman', :latest\n\nmod 'theforeman/puppet', :latest\n\nmod 'theforeman/foreman_proxy', :latest\n\nmod 'puppet/nftables', :latest\n\nmod 'herculesteam/augeasproviders_sysctl', :latest\n\nmod 'camptocamp/catalog_diff', :latest\n\nmod 'puppet/unattended_upgrades', :latest\n\nmod 'puppet/selinux', :latest\n\nmod 'choria/choria', :latest\n\nmod 'puppet/archive', :latest\n\nmod 'puppet/elasticsearch', :latest\n\nmod 'jsok/vault',\n\n  git: 'https://github.com/bastelfreak/puppet-vault',\n\n  ref: 'test'\n\n\n\nmod 'herculesteam/augeasproviders_core', :latest\n\nmod 'puppetlabs/stdlib', :latest\n\nmod 'choria/mcollective', :latest\n\nmod 'puppetlabs/concat', :latest\n\nmod 'puppetlabs/apt', :latest\n\nmod 'puppetlabs/apache', :latest\n\nmod 'puppetlabs/postgresql', :latest\n\nmod 'puppet/extlib', :latest\n\nmod 'puppet/redis', :latest\n\nmod 'puppet/epel', :latest\n\nmod 'theforeman/puppetserver_foreman', :latest\n\nmod 'richardc/datacat', :latest\n\nmod 'theforeman/dns', :latest\n\nmod 'theforeman/dhcp', :latest\n\nmod 'theforeman/tftp', :latest\n\nmod 'puppetlabs/xinetd', :latest\n\nmod 'choria-mcollective_choria', :latest\n\nmod 'choria/mcollective_agent_puppet', :latest\n\nmod 'choria/mcollective_agent_package', :latest\n\nmod 'choria/mcollective_agent_service', :latest\n\nmod 'choria/mcollective_agent_filemgr', :latest\n\nmod 'choria/mcollective_agent_shell', :latest\n\nmod 'choria/mcollective_agent_nettest', :latest\n\nmod 'choria/mcollective_agent_puppetca', :latest\n\nmod 'choria/mcollective_agent_bolt_tasks', :latest\n\nmod 'choria/mcollective_agent_iptables', :latest\n\nmod 'choria/mcollective_agent_process', :latest\n\nmod 'choria/mcollective_util_actionpolicy', :latest\n\nmod 'optiz0r/mcollective_agent_puppet_env', :latest\n\nmod 'jay7x/mcollective_agent_query', :latest\n\nmod 'choria/mcollective_data_sysctl', :latest\n\nmod 'puppet/yum', :latest\n\nmod 'puppetlabs/java', :latest\n\nmod 'puppet/elastic_stack', :latest\n\nmod 'puppet/hashi_stack', :latest\n\n\n\nmod 'puppetlabs-sshkeys_core', :latest\n\nmod 'puppetlabs-mount_core', :latest\n\nmod 'puppetlabs-augeas_core', :latest\n\nmod 'puppetlabs-selinux_core', :latest\n\nmod 'puppetlabs-yumrepo_core', :latest\n"
irb(main):006:0>

[logicminds]

Ok, so line 31 causes empty quotes which is not removed in the parser

all_lines = File.read(puppetfile).lines.map(&:strip_comment) This line leaves empty quotes when it should not.