akash-network / support

Akash Support and Issue Tracking
Apache License 2.0
5 stars 4 forks source link

sandbox: testing 0.28.2 -> 0.29.0-rc0 upgrade #151

Closed andy108369 closed 10 months ago

andy108369 commented 10 months ago

upgrade done via binary swap

blocks

root@sandbox-validator-1:~# date; docker compose -f sandbox.yaml logs validator1 -f --tail=100 ; date
Wed Nov 22 10:10:50 UTC 2023
root-validator1-1  | 10:09AM INF pre-upgrade command does not exist. continuing the upgrade. module=cosmovisor
root-validator1-1  | 10:09AM INF upgrade detected, relaunching app=akash module=cosmovisor
root-validator1-1  | 10:09AM INF running app args=["start"] module=cosmovisor path=/root/.akash/cosmovisor/upgrades/v0.28.0/bin/akash
root-validator1-1  | INF initializing upgrade `v0.28.0`
root-validator1-1  | INF applying store upgrades for `v0.28.0`
root-validator1-1  | INF initializing upgrade `v0.30.0`
root-validator1-1  | INF initializing upgrade `v0.24.0`
root-validator1-1  | INF initializing upgrade `akash_v0.15.0_cosmos_v0.44.x`
root-validator1-1  | INF initializing upgrade `v0.20.0`
root-validator1-1  | INF initializing upgrade `v0.26.0`
root-validator1-1  | INF starting node with ABCI Tendermint in-process
root-validator1-1  | INF service start impl=multiAppConn module=proxy msg={}
root-validator1-1  | INF service start connection=query impl=localClient module=abci-client msg={}
root-validator1-1  | INF service start connection=snapshot impl=localClient module=abci-client msg={}
root-validator1-1  | INF service start connection=mempool impl=localClient module=abci-client msg={}
root-validator1-1  | INF service start connection=consensus impl=localClient module=abci-client msg={}
root-validator1-1  | INF service start impl=EventBus module=events msg={}
root-validator1-1  | INF service start impl=PubSub module=pubsub msg={}
root-validator1-1  | INF service start impl=IndexerService module=txindex msg={}
root-validator1-1  | INF ABCI Handshake App Info hash="XQ�\nsUJ��v4�\x11\x10ͻ^O�B�'����\"�4C��" height=1981379 module=consensus protocol-version=0 software-version=v0.29.0-rc0
root-validator1-1  | INF ABCI Replay Blocks appHeight=1981379 module=consensus stateHeight=1981379 storeHeight=1981379
root-validator1-1  | INF Completed ABCI Handshake - CometBFT and App are synced appHash="XQ�\nsUJ��v4�\x11\x10ͻ^O�B�'����\"�4C��" appHeight=1981379 module=consensus
root-validator1-1  | INF Version info abci=0.17.0 block=11 cmtbft_version=0.34.27 commit_hash= p2p=8
root-validator1-1  | INF This node is a validator addr=5F2AE3322D6855B43FB17CDFB35E3A9C8E4FBDC3 module=consensus pubKey=ezVYNinrfDOheyueAxXD1ESIRG79I89lX2sY6pPdyN4=
root-validator1-1  | INF P2P Node ID ID=130dd841be22674b5d2840827a86386652a2d96a file=/root/.akash/config/node_key.json module=p2p
root-validator1-1  | INF Adding persistent peers addrs=["05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667@validator2:26656","3a0f6ca48168538663eb06e7fa36b1ce09a771f0@validator3:26656"] module=p2p
root-validator1-1  | INF Add our address to book addr={"id":"130dd841be22674b5d2840827a86386652a2d96a","ip":"0.0.0.0","port":26656} book=/root/.akash/config/addrbook.json module=p2p
root-validator1-1  | INF service start impl=Node msg={}
root-validator1-1  | INF Starting pprof server laddr=localhost:6060
root-validator1-1  | INF service start impl="P2P Switch" module=p2p msg={}
root-validator1-1  | INF service start impl=BlockchainReactor module=blockchain msg={}
root-validator1-1  | INF service start impl=BlockPool module=blockchain msg={}
root-validator1-1  | INF serve module=rpc-server msg={}
root-validator1-1  | INF service start impl=ConsensusReactor module=consensus msg={}
root-validator1-1  | INF Reactor  module=consensus waitSync=true
root-validator1-1  | INF service start impl=Evidence module=evidence msg={}
root-validator1-1  | INF service start impl=StateSync module=statesync msg={}
root-validator1-1  | INF service start impl=PEX module=pex msg={}
root-validator1-1  | INF service start book=/root/.akash/config/addrbook.json impl=AddrBook module=p2p msg={}
root-validator1-1  | INF Ensure peers module=pex numDialing=0 numInPeers=0 numOutPeers=0 numToDial=10
root-validator1-1  | ERR Error in peer's address err="error looking up host (validator2): lookup validator2 on 127.0.0.11:53: server misbehaving" module=p2p
root-validator1-1  | ERR Error in peer's address err="error looking up host (validator3): lookup validator3 on 127.0.0.11:53: server misbehaving" module=p2p
root-validator1-1  | INF Saving AddrBook to file book=/root/.akash/config/addrbook.json module=p2p size=5
root-validator1-1  | INF starting API server... module=api-server
root-validator1-1  | INF serve module=api-server msg={}
root-validator1-1  | INF service start impl="Peer{MConn{172.17.0.6:38318} 05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667 in}" module=p2p msg={} peer={"id":"05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667","ip":"172.17.0.6","port":38318}
root-validator1-1  | INF service start impl=MConn{172.17.0.6:38318} module=p2p msg={} peer={"id":"05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667","ip":"172.17.0.6","port":38318}
root-validator1-1  | INF Time to switch to consensus reactor! height=1981380 module=blockchain
root-validator1-1  | INF service stop impl={"Logger":{}} module=blockchain msg={}
root-validator1-1  | INF SwitchToConsensus module=consensus
root-validator1-1  | INF service start impl=ConsensusState module=consensus msg={}
root-validator1-1  | INF service start impl=baseWAL module=consensus msg={} wal=/root/.akash/data/cs.wal/wal
root-validator1-1  | INF service start impl=Group module=consensus msg={} wal=/root/.akash/data/cs.wal/wal
root-validator1-1  | INF service start impl=TimeoutTicker module=consensus msg={}
root-validator1-1  | INF Searching for height height=1981380 max=784 min=682 module=consensus wal=/root/.akash/data/cs.wal/wal
root-validator1-1  | INF Searching for height height=1981379 max=784 min=682 module=consensus wal=/root/.akash/data/cs.wal/wal
root-validator1-1  | INF Found height=1981379 index=784 module=consensus wal=/root/.akash/data/cs.wal/wal
root-validator1-1  | INF Catchup by replaying consensus messages height=1981380 module=consensus
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepNewHeight
root-validator1-1  | INF Replay: Vote blockID={"hash":"E386F40CD0F89485E9FD96C849EC479F64FF3394E970B6D060C7FA0ECE601CA5","parts":{"hash":"A9E8A532DFD8F697DCFE8819294CA271C51C290A3480109A323C23222CB6F369","total":1}} height=1981379 module=consensus peer=05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667 round=0 type=2
root-validator1-1  | INF Replay: Timeout dur=4938.372571 height=1981380 module=consensus round=0 step=1
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepPropose
root-validator1-1  | INF Replay: Proposal header={"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1} height=1981380 module=consensus peer=local pol=-1 round=0
root-validator1-1  | INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}},"height":1981380,"pol_round":-1,"round":0,"signature":"TWxUFIetx0ton5PTUTYic7XhTdGsugTfmMWv3u7INIE7fihcs+3BVYl4WeS8y7ALLpUZHEta4G9fW/jT5ppWDg==","timestamp":"2023-11-22T10:07:42.689566859Z"}
root-validator1-1  | INF Replay: BlockPart height=1981380 module=consensus peer=local round=0
root-validator1-1  | INF received complete proposal block hash=BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA height=1981380 module=consensus
root-validator1-1  | ERR failed signing vote err="error signing vote: step regression at height 1981380 round 0. Got 2, last step 3" height=1981380 module=consensus round=0 vote={"block_id":{"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}},"height":1981380,"round":0,"signature":null,"timestamp":"2023-11-22T10:10:17.461171688Z","type":1,"validator_address":"5F2AE3322D6855B43FB17CDFB35E3A9C8E4FBDC3","validator_index":2}
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepPrevote
root-validator1-1  | INF Replay: Vote blockID={"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}} height=1981380 module=consensus peer=local round=0 type=1
root-validator1-1  | INF Replay: Vote blockID={"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}} height=1981380 module=consensus peer=05be3cd53c5fca5dec6f3bee21eb5f2d1ac9e667 round=0 type=1
root-validator1-1  | INF Replay: Vote blockID={"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}} height=1981380 module=consensus peer=3a0f6ca48168538663eb06e7fa36b1ce09a771f0 round=0 type=1
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepPrecommit
root-validator1-1  | INF Replay: Vote blockID={"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}} height=1981380 module=consensus peer=local round=0 type=2
root-validator1-1  | INF Replay: Vote blockID={"hash":"BBFAEFB69CA095145AD7AFD142BB771763FD8B4DFE6CD11C2FE95871EAA746EA","parts":{"hash":"0C92F10B5AF26F6FA60B51681D457F936A902D4FD0CB1B3C71DC291FF4B1ACF5","total":1}} height=1981380 module=consensus peer=3a0f6ca48168538663eb06e7fa36b1ce09a771f0 round=0 type=1
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepPropose
root-validator1-1  | INF Replay: New Step height=1981380 module=consensus round=0 step=RoundStepPrevote
root-validator1-1  | INF Replay: Done module=consensus
root-validator1-1  | INF Timed out dur=3000 height=1981380 module=consensus round=0 step=3
root-validator1-1  | INF Ensure peers module=pex numDialing=0 numInPeers=0 numOutPeers=0 numToDial=10
root-validator1-1  | INF We need more addresses. Sending pexRequest to random peer module=pex peer={"Data":{},"Logger":{}}
root-validator1-1  | INF finalizing commit of block hash={} height=1981380 module=consensus num_txs=0 root=5851EB0A73554AE6A87634A81110CDBB5E4F93429B27A0BCCAF122DA3443C9CA
root-validator1-1  | INF minted coins from module account amount=2809098uakt from=mint module=x/bank
root-validator1-1  | INF executed block height=1981380 module=state num_invalid_txs=0 num_valid_txs=0
root-validator1-1  | INF commit synced commit=436F6D6D697449447B5B31303920323520313220313034203231312032353020313937203020343320363520353320323438203536203232322031363520313430203536203231302032362032303620383820382032302037342032342031333720313634203637203135322031303220313033203132375D3A3145334243347D
root-validator1-1  | INF committed state app_hash=6D190C68D3FAC5002B4135F838DEA58C38D21ACE5808144A1889A4439866677F height=1981380 module=state num_txs=0
root-validator1-1  | INF indexed block exents height=1981380 module=txindex
root-validator1-1  | INF service start impl="Peer{MConn{172.17.0.7:36172} 3a0f6ca48168538663eb06e7fa36b1ce09a771f0 in}" module=p2p msg={} peer={"id":"3a0f6ca48168538663eb06e7fa36b1ce09a771f0","ip":"172.17.0.7","port":36172}
root-validator1-1  | INF service start impl=MConn{172.17.0.7:36172} module=p2p msg={} peer={"id":"3a0f6ca48168538663eb06e7fa36b1ce09a771f0","ip":"172.17.0.7","port":36172}
root-validator1-1  | INF Timed out dur=4878.504539 height=1981381 module=consensus round=0 step=1
root-validator1-1  | INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"35822CB4963D3AE8595555EBC9194AE76DFF6228B1052396255A5D07B736EB06","parts":{"hash":"3E92E1E163CE654BBC410D8070E81FD4140928B8D012368D13FFB62DD46F6B8E","total":1}},"height":1981381,"pol_round":-1,"round":0,"signature":"mGuG8Y+fKRYNA2rqwurogjbNTFeKYIskMWyuXMpcpvEaGZy5XVQV4z1SyZVGh9Jabpt65ENU5AUkhMAe6XbYDQ==","timestamp":"2023-11-22T10:10:47.572720677Z"}
root-validator1-1  | INF received complete proposal block hash=35822CB4963D3AE8595555EBC9194AE76DFF6228B1052396255A5D07B736EB06 height=1981381 module=consensus
root-validator1-1  | INF finalizing commit of block hash={} height=1981381 module=consensus num_txs=0 root=6D190C68D3FAC5002B4135F838DEA58C38D21ACE5808144A1889A4439866677F
root-validator1-1  | INF minted coins from module account amount=2809099uakt from=mint module=x/bank
root-validator1-1  | INF executed block height=1981381 module=state num_invalid_txs=0 num_valid_txs=0
root-validator1-1  | INF commit synced commit=436F6D6D697449447B5B3131392031373120313032203232352031332032323420313535203735203635203135322031353920313335203136312032333520323239203934203130362032333320363620313731203931203132312038342036302032323120323231203138382034382037342035332038203131385D3A3145334243357D
root-validator1-1  | INF committed state app_hash=77AB66E10DE09B4B41989F87A1EBE55E6AE942AB5B79543CDDDDBC304A350876 height=1981381 module=state num_txs=0
root-validator1-1  | INF indexed block exents height=1981381 module=txindex
root-validator1-1  | INF Timed out dur=4933.595879 height=1981382 module=consensus round=0 step=1
root-validator1-1  | INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"A75654918013DE390A658ACE210E68904D8BD1340E91651A2F65CB35D64DC697","parts":{"hash":"BAE0A6E81639C5060A09F5F1488F428A127FF60580EE6DA38DC6E3ED67AA5DF8","total":1}},"height":1981382,"pol_round":-1,"round":0,"signature":"8dM+P82Mxo1WMkNFzpzr4p1Y2cSYrwekHIB9c9wNbtCzQyFWuPZwdxDdcd2wRQCbaDXV80oz+4COexHaE8/6Dg==","timestamp":"2023-11-22T10:10:52.932434126Z"}
root-validator1-1  | INF received complete proposal block hash=A75654918013DE390A658ACE210E68904D8BD1340E91651A2F65CB35D64DC697 height=1981382 module=consensus
root-validator1-1  | INF finalizing commit of block hash={} height=1981382 module=consensus num_txs=0 root=77AB66E10DE09B4B41989F87A1EBE55E6AE942AB5B79543CDDDDBC304A350876
root-validator1-1  | INF minted coins from module account amount=2809099uakt from=mint module=x/bank
root-validator1-1  | INF executed block height=1981382 module=state num_invalid_txs=0 num_valid_txs=0
root-validator1-1  | INF commit synced commit=436F6D6D697449447B5B3234302034322031303620313935203631203535203130372036382039322032323920373920323520313133203135342031343520313633203137203231352032303320313536203130392032353120313236203138352033312031323020313238203138372031373320313035203336203130355D3A3145334243367D
root-validator1-1  | INF committed state app_hash=F02A6AC33D376B445CE54F19719A91A311D7CB9C6DFB7EB91F7880BBAD692469 height=1981382 module=state num_txs=0
root-validator1-1  | INF indexed block exents height=1981382 module=txindex
root-validator1-1  | INF Ensure peers module=pex numDialing=0 numInPeers=0 numOutPeers=0 numToDial=10
root-validator1-1  | INF We need more addresses. Sending pexRequest to random peer module=pex peer={"Data":{},"Logger":{}}
root-validator1-1  | INF Timed out dur=4918.923948 height=1981383 module=consensus round=0 step=1
root-validator1-1  | INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"36CBE6528F424FD4391828E26F6D23B8D9A36EB50E908A9327CE11F3F66B855C","parts":{"hash":"DC130359DE43A28AED3AAFF9B06EDE8D2831380BC04567F0184BA3737A8AAB75","total":1}},"height":1981383,"pol_round":-1,"round":0,"signature":"7UoAan/3ichDusQziuRRWsjT33Od5mNRmNokGPq8vpblQZtEWHqAbFAdwxCdwbmTI6iVdT7FBkGn8YZUE7dIDg==","timestamp":"2023-11-22T10:10:58.37342926Z"}
root-validator1-1  | INF received complete proposal block hash=36CBE6528F424FD4391828E26F6D23B8D9A36EB50E908A9327CE11F3F66B855C height=1981383 module=consensus
root-validator1-1  | INF finalizing commit of block hash={} height=1981383 module=consensus num_txs=0 root=F02A6AC33D376B445CE54F19719A91A311D7CB9C6DFB7EB91F7880BBAD692469
root-validator1-1  | INF minted coins from module account amount=2809100uakt from=mint module=x/bank
root-validator1-1  | INF executed block height=1981383 module=state num_invalid_txs=0 num_valid_txs=0
root-validator1-1  | INF commit synced commit=436F6D6D697449447B5B3937203136203333203330203838203133342032353520363120393320323234203135392038352031393520313438203135302031383020323439203132203234203135322031323520313733203139312031313220312032333720313634203932203233392032342031372031335D3A3145334243377D
root-validator1-1  | INF committed state app_hash=6110211E5886FF3D5DE09F55C39496B4F90C18987DADBF7001EDA45CEF18110D height=1981383 module=state num_txs=0
...
andy108369 commented 10 months ago

provider bids, creates the lease, receives the manifest and spawns the deployment. lease-shell OK.

dseqs tested 1981418, 1981438.

andy108369 commented 10 months ago

provider bids the SDL with the mixed count value :rocket: i.e. it fixes https://github.com/akash-network/support/issues/149

I[2023-11-22|10:22:15.275] bid complete                                 module=bidengine-order cmp=provider order=akash1nx9pr8jee9jx44tkgt62fmgt2hmgvru92td3hg/1981505/1/1

SDL

version: '2.0'

services:
  foo:
    image: ubuntu:22.04
    command:
      - "sh"
      - "-c"
    args:
      - sleep infinity
    expose:
      - port: 80
        as: 80
        to: 
          - service: bar
          - global: true

  bar:
    image: ubuntu:22.04
    command:
      - "sh"
      - "-c"
    args:
      - sleep infinity
    expose:
      - port: 80
        as: 80
        to:
          - global: true

profiles:
  compute:
    foo:
      resources:
        cpu:
          units: 1
        memory:
          size: 1Gi
        storage:
          size: 1Gi

    bar:
      resources:
        cpu:
          units: 1
        memory:
          size: 1Gi
        storage:
          size: 1Gi

  placement:
    westcoast:
      pricing:
        foo:
          denom: uakt
          amount: 1000000
        bar:
          denom: uakt
          amount: 1000000

deployment:
  foo:
    westcoast:
      profile: foo
      count: 1
  bar:
    westcoast:
      profile: bar
      count: 2

provider /stats

before

$ provider_info.sh provider.provider-02.sandbox-01.aksh.pw
provider.provider-02.sandbox-01.aksh.pw  akash1rk090a6mq9gvm0h6ljf8kz8mrxglwwxsk4srxh                                               
type                                     cpu                                           gpu  ram                 ephemeral           persistent
used                                     10                                            0    10                  10                  0
pending                                  0.1                                           0    0.5                 0.5                 0
available                                4.55                                          2    20.132808685302734  163.78484315704554  30.318228708580136
node                                     4.55                                          2    20.132808685302734  163.78484315704554  N/A

after

$ provider_info.sh provider.provider-02.sandbox-01.aksh.pw
provider.provider-02.sandbox-01.aksh.pw  akash1rk090a6mq9gvm0h6ljf8kz8mrxglwwxsk4srxh                                               
type                                     cpu                                           gpu  ram                 ephemeral           persistent
used                                     13                                            0    13                  13                  0
pending                                  0                                             0    0                   0                   0
available                                1.65                                          2    17.632808685302734  161.28484315704554  30.318228708580136
node                                     1.65                                          2    17.632808685302734  161.28484315704554  N/A
andy108369 commented 10 months ago

also bids to:

dseq 1981560

deployment:
  foo:
    westcoast:
      profile: foo
      count: 1
  bar:
    westcoast:
      profile: bar
      count: 3

dseq 1981572

deployment:
  foo:
    westcoast:
      profile: foo
      count: 2
  bar:
    westcoast:
      profile: bar
      count: 2

dseq 1981582

deployment:
  foo:
    westcoast:
      profile: foo
      count: 1
  bar:
    westcoast:
      profile: bar
      count: 1

dseq 1981599

deployment:
  foo:
    westcoast:
      profile: foo
      count: 1
  bar:
    westcoast:
      profile: bar
      count: 1
  baz:
    westcoast:
      profile: baz
      count: 1

dseq 1981613

deployment:
  foo:
    westcoast:
      profile: foo
      count: 2
  bar:
    westcoast:
      profile: bar
      count: 1
  baz:
    westcoast:
      profile: baz
      count: 3
andy108369 commented 10 months ago

GPU SDL with mixed count works as well! :rocket:

version: '2.0'

services:
  foo:
    image: ubuntu:22.04
    command:
      - "sh"
      - "-c"
    args:
      - sleep infinity
    expose:
      - port: 80
        as: 80
        to: 
          - service: bar
          - global: true

  bar:
    image: ubuntu:22.04
    command:
      - "sh"
      - "-c"
    args:
      - sleep infinity
    expose:
      - port: 80
        as: 80
        to:
          - global: true

profiles:
  compute:
    foo:
      resources:
        cpu:
          units: 0.1
        memory:
          size: 1Gi
        storage:
          size: 1Gi

    bar:
      resources:
        cpu:
          units: 0.1
        memory:
          size: 1Gi
        storage:
          size: 1Gi
        gpu:
          units: 1
          attributes:
            vendor:
              nvidia:
                #- model: rtx8000

  placement:
    westcoast:
      pricing:
        foo:
          denom: uakt
          amount: 1000000
        bar:
          denom: uakt
          amount: 1000000

deployment:
  foo:
    westcoast:
      profile: foo
      count: 1
  bar:
    westcoast:
      profile: bar
      count: 2

and it picked two GPU's as expected:

$ provider_info.sh provider.provider-02.sandbox-01.aksh.pw
provider.provider-02.sandbox-01.aksh.pw  akash1rk090a6mq9gvm0h6ljf8kz8mrxglwwxsk4srxh                                               
type                                     cpu                                           gpu  ram                 ephemeral           persistent
used                                     10                                            0    10                  10                  0
pending                                  0.3                                           2    3                   3                   0
available                                4.35                                          0    17.632808685302734  161.28484315704554  30.318228708580136
node                                     4.35                                          0    17.632808685302734  161.28484315704554  N/A