Expose img2dataset distributor

0x2b3bfa0 commented 1 year ago

rom1504 commented 1 year ago

Cool!

On Fri, Sep 15, 2023, 11:28 Helio Machado @.***> wrote:

Note to self

cluster_name: dvcx-datacomp-downloadermin_workers: 0max_workers: 10upscaling_speed: 1.0available_node_types: ray.head.default: resources: {} node_config: ImageId: ami-0d8783a0bfd0c4db1 # ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230106 InstanceType: m5.8xlarge BlockDeviceMappings:

DeviceName: /dev/sda1 Ebs: DeleteOnTermination: true VolumeSize: 64 # GB VolumeType: gp2 ray.worker.default: min_workers: 0 max_workers: 500 resources: {} node_config: ImageId: ami-0d8783a0bfd0c4db1 # ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230106 InstanceType: m5.12xlarge BlockDeviceMappings:

DeviceName: /dev/sda1 Ebs: DeleteOnTermination: true VolumeSize: 64 # GB VolumeType: gp2 provider: type: aws region: us-east-2 cache_stopped_nodes: false file_mounts: .: . initialization_commands:

wget https://secure.nic.cz/files/knot-resolver/knot-resolver-release.deb

sudo DEBIAN_FRONTEND=noninteractive apt install ./knot-resolver-release.deb && sudo apt update

sudo DEBIAN_FRONTEND=noninteractive apt install --yes knot-resolver ffmpeg libsm6 libxext6 build-essential

echo $(hostname -I) $(hostname) | sudo tee --append /etc/hosts

echo nameserver 127.0.0.1 | sudo tee /etc/resolv.conf

sudo systemctl stop systemd-resolved

sudo systemctl start kresd@{1..8}.service setup_commands:

sudo mkdir --parents /opt/miniconda3 && sudo chown ubuntu:ubuntu /opt/miniconda3

wget https://repo.anaconda.com/miniconda/Miniconda3-py39_22.11.1-1-Linux-x86_64.sh -O /opt/miniconda3/install.sh

bash /opt/miniconda3/install.sh -f -b -p /opt/miniconda3

echo 'export PATH="/opt/miniconda3/bin/:$PATH"' >> ~/.bashrc

echo 'export AWS_SECRET_ACCESS_KEY=...' >> ~/.bashrc

echo 'export AWS_ACCESS_KEY_ID=...' >> ~/.bashrc

echo 'export AWS_SESSION_TOKEN=...' >> ~/.bashrc

conda env create --file environment.yml && conda init

conda activate datacomp && pip install --upgrade 'ray[default]'

conda activate datacomp && pip install s3fs 'cloudpathlib[s3]' head_setup_commands: [] head_start_ray_commands:

conda activate datacomp && ray stop

conda activate datacomp && ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host=0.0.0.0 worker_start_ray_commands:

conda activate datacomp && ray stop

conda activate datacomp && ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076

— Reply to this email directly, view it on GitHub https://github.com/mlfoundations/datacomp/pull/58#issuecomment-1720965746, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAR437QVQ6HVFACR6BPAXLLX2QNTHANCNFSM6AAAAAA4ZNB37A . You are receiving this because you are subscribed to this thread.Message ID: @.***>

rom1504 commented 1 year ago

@Vaishaal fyi

0x2b3bfa0 commented 9 months ago

@Vaishaal & @rom1504, ping 🛎️

mlfoundations / datacomp

Expose img2dataset distributor #58