TODO: lib/whatweb/scan: refactor url list parsing

[bcoles]

      # TODO: refactor this
      url_list = url_list.map do |x|
        if File.exist?(x)
          x
        else
          # use url pattern
          x = opts[:url_pattern].gsub('%insert%', x) unless opts[:url_pattern].to_s.eql?('')
          # add prefix & suffix
          x = "#{opts[:url_prefix]}#{x}#{opts[:url_suffix]}"

          # need to move this into a URI parsing function
          #
          # check for URI prefix
          if x !~ %r{^[a-z]+:\/\/}
            # add missing URI prefix
            x.sub!(/^/, 'http://')
          end

          # is it a valid domain?
          begin
            domain = Addressable::URI.parse(x)
            # check validity
            raise 'Unable to parse invalid target. No hostname.' if domain.host.empty?

            # convert IDN domain
            x = domain.normalize.to_s if domain.host !~ %r{^[a-zA-Z0-9\.:\/]*$}
          rescue => e
            # if it fails it's not valid
            x = nil
            # TODO: print something more useful
            error("Unable to parse invalid target #{x}: #{e}")
          end
          # return x
          x
        end
      end