Open AlexWortega opened 3 years ago
@AlexWortega could you use `` at the begin and the end of your code ? You can use the tab
Preview` to check the display.
It's hard to read your code :-/
Could you also tell us what did you change ?
I would like to use all my GPU to.
I think you only need to add DataParallel to the top level model, rather than in multiple places in the model implementation. this tutorial suggests something like:
diff --git a/big_sleep/cli.py b/big_sleep/cli.py
index 8aa8820..a4cbdf4 100644
--- a/big_sleep/cli.py
+++ b/big_sleep/cli.py
@@ -4,6 +4,7 @@ from big_sleep import Imagine, version
from pathlib import Path
from .version import __version__;
+import torch
def train(
text=None,
@@ -68,6 +69,11 @@ def train(
if answer not in ('yes', 'y'):
exit()
+ if torch.cuda.device_count() > 1:
+ print("Let's use", torch.cuda.device_count(), "GPUs!")
+ imagine = torch.nn.DataParallel(imagine)
+ imagine.to(torch.device('cuda'))
+
imagine()
def main():
ought to allow the dream
script to use multiple GPUs. However, this model get an exception:
RuntimeError: Output 237 of BroadcastBackward is a view and its base or another view of its base has been modified inplace. This view is the output of a function that returns multiple views. Such functions do not allow the output views to be modified inplace. You should replace the inplace operation by an out-of-place one.
and I think that means this architecture needs a little retooling to support multi-gpu training.
Hi ive tried to add multiply gpu support by nn.DataParallel Now its looks like this ` import os import sys import subprocess import signal import string import re
from datetime import datetime from pathlib import Path import random
import torch import torch.nn.functional as F from torch import nn from torch.optim import Adam from torchvision.utils import save_image import torchvision.transforms as T from PIL import Image from tqdm import tqdm, trange
from ema import EMA from resample import resample from biggan import BigGAN from clip import load, tokenize
assert torch.cuda.is_available(), 'CUDA must be available in order to use Big Sleep'
graceful keyboard interrupt
terminate = False
def signal_handling(signum,frame): global terminate terminate = True
signal.signal(signal.SIGINT,signal_handling)
helpers
def exists(val): return val is not None
def open_folder(path): if os.path.isfile(path): path = os.path.dirname(path)
def create_text_path(text=None, img=None, encoding=None): input_name = "" if text is not None: input_name += text if img is not None: if isinstance(img, str): img_name = "".join(img.split(".")[:-1]) # replace spaces by underscores, remove img extension img_name = img_name.split("/")[-1] # only take img name, not path else: img_name = "PIL_img" inputname += "" + img_name if encoding is not None: input_name = "your_encoding" return inputname.replace("-", "").replace(",", "").replace(" ", "").replace("|", "--").strip('-')[:255]
tensor helpers
def differentiable_topk(x, k, temperature=1.): n, dim = x.shape topk_tensors = []
def create_clip_img_transform(image_width): clip_mean = [0.48145466, 0.4578275, 0.40821073] clip_std = [0.26862954, 0.26130258, 0.27577711] transform = T.Compose([
T.ToPILImage(),
def rand_cutout(image, size, center_bias=False, center_focus=2): width = image.shape[-1] min_offset = 0 max_offset = width - size if center_bias:
sample around image center
load clip
perceptor, normalize_image = load('ViT-B/32', jit = False)
load biggan
class Latents(torch.nn.Module): def init( self, num_latents = 15, num_classes = 1000, z_dim = 128, max_classes = None, class_temperature = 2. ): super().init() self.normu = torch.nn.Parameter(torch.zeros(num_latents, zdim).normal(std = 1)) self.cls = torch.nn.Parameter(torch.zeros(num_latents, numclasses).normal(mean = -3.9, std = .3)) self.register_buffer('thresh_lat', torch.tensor(1))
class Model(nn.Module): def init( self, image_size, max_classes = None, class_temperature = 2., ema_decay = 0.99 ): super().init() assert image_size in (128, 256, 512), 'image size must be one of 128, 256, or 512' self.biggan = BigGAN.from_pretrained(f'biggan-deep-{image_size}') self.max_classes = max_classes self.class_temperature = class_temperature self.ema_decay\ = ema_decay
class BigSleep(nn.Module): def init( self, num_cutouts = 128, loss_coef = 100, image_size = 512, bilinear = False, max_classes = None, class_temperature = 2., experimental_resample = False, ema_decay = 0.99, center_bias = False, ): super().init() self.loss_coef = loss_coef self.image_size = image_size self.num_cutouts = num_cutouts self.experimental_resample = experimental_resample self.center_bias = center_bias
class Imagine(nn.Module): def init( self, *, text=None, img=None, encoding=None, text_min = "", lr = .07, image_size = 512, gradient_accumulate_every = 1, save_every = 50, epochs = 20, iterations = 1050, save_progress = False, bilinear = False, open_folder = True, seed = None, append_seed = False, torch_deterministic = False, max_classes = None, class_temperature = 2., save_date_time = False, save_best = False, experimental_resample = False, ema_decay = 0.99, num_cutouts = 128, center_bias = False, ): super().init()
`
But i recive this error `UserWarning: 'torch.load' received a zip file that looks like a TorchScript archive dispatching to 'torch.jit.load' (call 'torch.jit.load' directly to silence this warning) " silence this warning)", UserWarning) Traceback (most recent call last): File "/home/jovyan/SberArtist/big-sleep/big_sleep/clip.py", line 101, in load model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval() File "/home/user/conda/lib/python3.7/site-packages/torch/jit/init.py", line 275, in load cpp_module = torch._C.import_ir_module(cu, f, map_location, _extra_files) RuntimeError:
aten::_convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups, bool benchmark, bool deterministic, bool cudnn_enabled) -> (Tensor): Expected at most 12 arguments but found 13 positional arguments.`