using delegation token

[JiHyunSong]

when delegation token renew hr settled, using delegation token when CREATE or APPEND

[JiHyunSong]

give me feedback plz 🙏

[tagomoris]

I want the change of README in addition to the code.

[tagomoris]

@JiHyunSong Are you trying this change in your environment? If true, releasing the new version will be easier for me.

[JiHyunSong]

@tagomoris works good in my sight

I inject some code for debugging I hope that my test scenario help you.

Really thank you for your careful feedback. 💯 It was fun and good experience :D

diff --git a/lib/webhdfs/client_v1.rb b/lib/webhdfs/client_v1.rb
index 0f4ff3e..f971409 100644
--- a/lib/webhdfs/client_v1.rb
+++ b/lib/webhdfs/client_v1.rb
@@ -29,6 +29,7 @@ module WebHDFS
     attr_accessor :kerberos, :kerberos_keytab
     attr_accessor :http_headers
     attr_accessor :kerberos_delegation_token
+    attr_accessor :kerberos_token_updated_at

     SSL_VERIFY_MODES = [:none, :peer]
     def ssl_verify_mode=(mode)
@@ -75,9 +76,11 @@ module WebHDFS
       return @kerberos_delegation_token if @kerberos_delegation_token && !should_kerberos_token_updated? && !force_renew

       if !@kerberos_delegation_token || force_renew
+        puts "call get_kerberos_delegation_token"
         @kerberos_delegation_token = get_kerberos_delegation_token(@username)
         @kerberos_token_updated_at = Time.now
       else
+        puts "call renew_kerberos_delegation_token"
         renew_kerberos_delegation_token(@kerberos_delegation_token)
         @kerberos_token_updated_at = Time.now
       end

test scenario

I will blind some value

irb(main):001:0> require 'webhdfs'
=> true
irb(main):002:0> host = 'hadoop-namenode'
=> "XX"
irb(main):004:0> port = '50070'
=> "50070"
irb(main):005:0> username = 'username'
=> "username"
irb(main):006:0>
irb(main):007:0* client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, 0.5)
=> #<WebHDFS::Client:0x00000000011495f0 @host="XX", @port="50070", @username="XX", @doas=nil, @proxy_address=nil, @proxy_port=nil, @retry_known_errors=false, @retry_times=1, @retry_interval=1, @httpfs_mode=false, @ssl=false, @ssl_ca_file=nil, @ssl_verify_mode=nil, @ssl_cert=nil, @ssl_key=nil, @ssl_version=nil, @kerberos=false, @kerberos_keytab=nil, @renew_kerberos_delegation_token_time_hour=0.5, @kerberos_delegation_token=nil, @kerberos_token_updated_at=2021-07-30 18:50:21 +0900, @http_headers={}>
irb(main):008:0> client.kerberos = true
=> true
irb(main):009:0> client.kerberos_keytab = 'my_keytab_path'
=> "XX"

# renew do not change token. just extend TTL
irb(main):013:0* first_token = client.get_cached_kerberos_delegation_token
call get_kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr20mUxigF7Gt7pMY0BCCCOAYEUwjlaTOpi29FGfFxGBpW3E063CgYSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"
irb(main):014:0> client.renew_kerberos_delegation_token(client.kerberos_delegation_token)
=> 1627725021583
irb(main):015:0> second_token_expected_same_first = client.get_cached_kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr20mUxigF7Gt7pMY0BCCCOAYEUwjlaTOpi29FGfFxGBpW3E063CgYSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"
irb(main):016:0>
irb(main):017:0* first_token == second_token_expected_same_first
=> true
irb(main):018:0> client.kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr20mUxigF7Gt7pMY0BCCCOAYEUwjlaTOpi29FGfFxGBpW3E063CgYSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"

# test cache refresh renew
irb(main):022:0> very_old_date = Time.utc(2000, "jan", 1, 20, 15, 1)
=> 2000-01-01 20:15:01 UTC
irb(main):023:0> client.should_kerberos_token_updated?
=> false
irb(main):024:0> client.kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr20mUxigF7Gt7pMY0BCCCOAYEUwjlaTOpi29FGfFxGBpW3E063CgYSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"
irb(main):025:0> client.kerberos_token_updated_at = very_old_date
=> 2000-01-01 20:15:01 UTC
irb(main):026:0> client.should_kerberos_token_updated?
=> true
irb(main):027:0> client.renew_kerberos_delegation_token(client.kerberos_delegation_token)
=> 1627725171649
irb(main):028:0> client.kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr20mUxigF7Gt7pMY0BCCCOAYEUwjlaTOpi29FGfFxGBpW3E063CgYSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"

# test old created token
irb(main):031:0* created_0729164205 = 'LwAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXrxNpj0igF7FUMc9I0WvVdmFHIqMKSk0gfwa_e7VcitiP7F2A6LEldFQkhERlMgZGVsZWdhdGlvbg8xMC45MC4zLjMxOjgwMjA'
=> "LwAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXrxNpj0igF7FUMc9I0WvVdmFHIqMKSk0gfwa_e7VcitiP7F2A6LEldFQkhERlMgZGVsZWdhdGlvbg8xMC45MC4zLjMxOjgwMjA" # this is expired on namenode
irb(main):032:0> client.renew_kerberos_delegation_token(created_0729164205)
call get_kerberos_delegation_token
=> 1627725207838
irb(main):033:0> client.kerberos_delegation_token
=> "MQAMam9hbm5lLWh3YW5nDGpvYW5uZS1od2FuZwCKAXr21TjXigF7GuG8140BCCGOAYEUikWyUMYhQscr1W1a45V5feyCzoUSV0VCSERGUyBkZWxlZ2F0aW9uEDEwLjkwLjI3LjI1OjgwMjA"
irb(main):034:0> first_token == client.kerberos_delegation_token
=> false

after update comments,

I reset the irb and test create file first debug message appears when first call

irb(main):013:0> data = 'hello delegation'
=> "hello delegation"
irb(main):014:0> client.create(hdfs_path + test_file, data)
call get_kerberos_delegation_token
=> true
irb(main):015:0> ^C
irb(main):015:0> client.content_summary(hdfs_path + test_file)
=> {"directoryCount"=>0, "fileCount"=>1, "length"=>16, "quota"=>-1, "spaceConsumed"=>48, "spaceQuota"=>-1, "typeQuota"=>{}}
irb(main):016:0> client.read(hdfs_path + test_file)
=> "hello delegation"

[tagomoris]

Just shipped v0.10.0. Thank you!

[JiHyunSong]

@tagomoris Thank you for your fast feedback :D